Fix misspellings of "system", "controller", "interrupt" and "necessary".
[linux-2.6.git] / drivers / ata / ahci.c
index 413270b..47c8060 100644 (file)
 #include <linux/blkdev.h>
 #include <linux/delay.h>
 #include <linux/interrupt.h>
-#include <linux/sched.h>
 #include <linux/dma-mapping.h>
 #include <linux/device.h>
 #include <scsi/scsi_host.h>
 #include <scsi/scsi_cmnd.h>
 #include <linux/libata.h>
-#include <asm/io.h>
 
 #define DRV_NAME       "ahci"
-#define DRV_VERSION    "2.0"
+#define DRV_VERSION    "3.0"
 
 
 enum {
        AHCI_PCI_BAR            = 5,
+       AHCI_MAX_PORTS          = 32,
        AHCI_MAX_SG             = 168, /* hardware max is 64K */
        AHCI_DMA_BOUNDARY       = 0xffffffff,
-       AHCI_USE_CLUSTERING     = 0,
+       AHCI_USE_CLUSTERING     = 1,
        AHCI_MAX_CMDS           = 32,
        AHCI_CMD_SZ             = 32,
        AHCI_CMD_SLOT_SZ        = AHCI_MAX_CMDS * AHCI_CMD_SZ,
@@ -74,11 +73,14 @@ enum {
        AHCI_CMD_CLR_BUSY       = (1 << 10),
 
        RX_FIS_D2H_REG          = 0x40, /* offset of D2H Register FIS data */
+       RX_FIS_SDB              = 0x58, /* offset of SDB FIS data */
        RX_FIS_UNK              = 0x60, /* offset of Unknown FIS data */
 
        board_ahci              = 0,
        board_ahci_vt8251       = 1,
        board_ahci_ign_iferr    = 2,
+       board_ahci_sb600        = 3,
+       board_ahci_mv           = 4,
 
        /* global controller registers */
        HOST_CAP                = 0x00, /* host capabilities */
@@ -94,8 +96,10 @@ enum {
 
        /* HOST_CAP bits */
        HOST_CAP_SSC            = (1 << 14), /* Slumber capable */
+       HOST_CAP_PMP            = (1 << 17), /* Port Multiplier support */
        HOST_CAP_CLO            = (1 << 24), /* Command List Override support */
        HOST_CAP_SSS            = (1 << 27), /* Staggered Spin-up */
+       HOST_CAP_SNTF           = (1 << 29), /* SNotification register */
        HOST_CAP_NCQ            = (1 << 30), /* Native Command Queueing */
        HOST_CAP_64             = (1 << 31), /* PCI DAC (64-bit DMA) support */
 
@@ -110,11 +114,11 @@ enum {
        PORT_TFDATA             = 0x20, /* taskfile data */
        PORT_SIG                = 0x24, /* device TF signature */
        PORT_CMD_ISSUE          = 0x38, /* command issue */
-       PORT_SCR                = 0x28, /* SATA phy register block */
        PORT_SCR_STAT           = 0x28, /* SATA phy register: SStatus */
        PORT_SCR_CTL            = 0x2c, /* SATA phy register: SControl */
        PORT_SCR_ERR            = 0x30, /* SATA phy register: SError */
        PORT_SCR_ACT            = 0x34, /* SATA phy register: SActive */
+       PORT_SCR_NTF            = 0x3c, /* SATA phy register: SNotification */
 
        /* PORT_IRQ_{STAT,MASK} bits */
        PORT_IRQ_COLD_PRES      = (1 << 31), /* cold presence detect */
@@ -140,7 +144,8 @@ enum {
                                  PORT_IRQ_IF_ERR |
                                  PORT_IRQ_CONNECT |
                                  PORT_IRQ_PHYRDY |
-                                 PORT_IRQ_UNK_FIS,
+                                 PORT_IRQ_UNK_FIS |
+                                 PORT_IRQ_BAD_PMP,
        PORT_IRQ_ERROR          = PORT_IRQ_FREEZE |
                                  PORT_IRQ_TF_ERR |
                                  PORT_IRQ_HBUS_DATA_ERR,
@@ -150,6 +155,7 @@ enum {
 
        /* PORT_CMD bits */
        PORT_CMD_ATAPI          = (1 << 24), /* Device is ATAPI */
+       PORT_CMD_PMP            = (1 << 17), /* PMP attached */
        PORT_CMD_LIST_ON        = (1 << 15), /* cmd list DMA engine running */
        PORT_CMD_FIS_ON         = (1 << 14), /* FIS DMA engine running */
        PORT_CMD_FIS_RX         = (1 << 4), /* Enable FIS receive DMA engine */
@@ -164,11 +170,21 @@ enum {
        PORT_CMD_ICC_SLUMBER    = (0x6 << 28), /* Put i/f in slumber state */
 
        /* hpriv->flags bits */
-       AHCI_FLAG_MSI           = (1 << 0),
+       AHCI_HFLAG_NO_NCQ               = (1 << 0),
+       AHCI_HFLAG_IGN_IRQ_IF_ERR       = (1 << 1), /* ignore IRQ_IF_ERR */
+       AHCI_HFLAG_IGN_SERR_INTERNAL    = (1 << 2), /* ignore SERR_INTERNAL */
+       AHCI_HFLAG_32BIT_ONLY           = (1 << 3), /* force 32bit */
+       AHCI_HFLAG_MV_PATA              = (1 << 4), /* PATA port */
+       AHCI_HFLAG_NO_MSI               = (1 << 5), /* no PCI MSI */
+       AHCI_HFLAG_NO_PMP               = (1 << 6), /* no PMP */
 
        /* ap->flags bits */
-       AHCI_FLAG_NO_NCQ                = (1 << 24),
-       AHCI_FLAG_IGN_IRQ_IF_ERR        = (1 << 25), /* ignore IRQ_IF_ERR */
+       AHCI_FLAG_NO_HOTPLUG            = (1 << 24), /* ignore PxSERR.DIAG.N */
+
+       AHCI_FLAG_COMMON                = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
+                                         ATA_FLAG_MMIO | ATA_FLAG_PIO_DMA |
+                                         ATA_FLAG_ACPI_SATA | ATA_FLAG_AN,
+       AHCI_LFLAG_COMMON               = ATA_LFLAG_SKIP_D2H_BSY,
 };
 
 struct ahci_cmd_hdr {
@@ -187,25 +203,32 @@ struct ahci_sg {
 };
 
 struct ahci_host_priv {
-       unsigned long           flags;
-       u32                     cap;    /* cache of HOST_CAP register */
-       u32                     port_map; /* cache of HOST_PORTS_IMPL reg */
+       unsigned int            flags;          /* AHCI_HFLAG_* */
+       u32                     cap;            /* cap to use */
+       u32                     port_map;       /* port map to use */
+       u32                     saved_cap;      /* saved initial cap */
+       u32                     saved_port_map; /* saved initial port_map */
 };
 
 struct ahci_port_priv {
+       struct ata_link         *active_link;
        struct ahci_cmd_hdr     *cmd_slot;
        dma_addr_t              cmd_slot_dma;
        void                    *cmd_tbl;
        dma_addr_t              cmd_tbl_dma;
        void                    *rx_fis;
        dma_addr_t              rx_fis_dma;
+       /* for NCQ spurious interrupt analysis */
+       unsigned int            ncq_saw_d2h:1;
+       unsigned int            ncq_saw_dmas:1;
+       unsigned int            ncq_saw_sdb:1;
+       u32                     intr_mask;      /* interrupts to enable */
 };
 
-static u32 ahci_scr_read (struct ata_port *ap, unsigned int sc_reg);
-static void ahci_scr_write (struct ata_port *ap, unsigned int sc_reg, u32 val);
+static int ahci_scr_read(struct ata_port *ap, unsigned int sc_reg, u32 *val);
+static int ahci_scr_write(struct ata_port *ap, unsigned int sc_reg, u32 val);
 static int ahci_init_one (struct pci_dev *pdev, const struct pci_device_id *ent);
 static unsigned int ahci_qc_issue(struct ata_queued_cmd *qc);
-static irqreturn_t ahci_interrupt (int irq, void *dev_instance);
 static void ahci_irq_clear(struct ata_port *ap);
 static int ahci_port_start(struct ata_port *ap);
 static void ahci_port_stop(struct ata_port *ap);
@@ -214,13 +237,20 @@ static void ahci_qc_prep(struct ata_queued_cmd *qc);
 static u8 ahci_check_status(struct ata_port *ap);
 static void ahci_freeze(struct ata_port *ap);
 static void ahci_thaw(struct ata_port *ap);
+static void ahci_pmp_attach(struct ata_port *ap);
+static void ahci_pmp_detach(struct ata_port *ap);
 static void ahci_error_handler(struct ata_port *ap);
+static void ahci_vt8251_error_handler(struct ata_port *ap);
 static void ahci_post_internal_cmd(struct ata_queued_cmd *qc);
-static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg);
 static int ahci_port_resume(struct ata_port *ap);
+static unsigned int ahci_fill_sg(struct ata_queued_cmd *qc, void *cmd_tbl);
+static void ahci_fill_cmd_slot(struct ahci_port_priv *pp, unsigned int tag,
+                              u32 opts);
+#ifdef CONFIG_PM
+static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg);
 static int ahci_pci_device_suspend(struct pci_dev *pdev, pm_message_t mesg);
 static int ahci_pci_device_resume(struct pci_dev *pdev);
-static void ahci_remove_one (struct pci_dev *pdev);
+#endif
 
 static struct scsi_host_template ahci_sht = {
        .module                 = THIS_MODULE,
@@ -239,23 +269,19 @@ static struct scsi_host_template ahci_sht = {
        .slave_configure        = ata_scsi_slave_config,
        .slave_destroy          = ata_scsi_slave_destroy,
        .bios_param             = ata_std_bios_param,
-       .suspend                = ata_scsi_device_suspend,
-       .resume                 = ata_scsi_device_resume,
 };
 
 static const struct ata_port_operations ahci_ops = {
-       .port_disable           = ata_port_disable,
-
        .check_status           = ahci_check_status,
        .check_altstatus        = ahci_check_status,
        .dev_select             = ata_noop_dev_select,
 
        .tf_read                = ahci_tf_read,
 
+       .qc_defer               = sata_pmp_qc_defer_cmd_switch,
        .qc_prep                = ahci_qc_prep,
        .qc_issue               = ahci_qc_issue,
 
-       .irq_handler            = ahci_interrupt,
        .irq_clear              = ahci_irq_clear,
 
        .scr_read               = ahci_scr_read,
@@ -267,43 +293,100 @@ static const struct ata_port_operations ahci_ops = {
        .error_handler          = ahci_error_handler,
        .post_internal_cmd      = ahci_post_internal_cmd,
 
+       .pmp_attach             = ahci_pmp_attach,
+       .pmp_detach             = ahci_pmp_detach,
+
+#ifdef CONFIG_PM
+       .port_suspend           = ahci_port_suspend,
+       .port_resume            = ahci_port_resume,
+#endif
+
+       .port_start             = ahci_port_start,
+       .port_stop              = ahci_port_stop,
+};
+
+static const struct ata_port_operations ahci_vt8251_ops = {
+       .check_status           = ahci_check_status,
+       .check_altstatus        = ahci_check_status,
+       .dev_select             = ata_noop_dev_select,
+
+       .tf_read                = ahci_tf_read,
+
+       .qc_defer               = sata_pmp_qc_defer_cmd_switch,
+       .qc_prep                = ahci_qc_prep,
+       .qc_issue               = ahci_qc_issue,
+
+       .irq_clear              = ahci_irq_clear,
+
+       .scr_read               = ahci_scr_read,
+       .scr_write              = ahci_scr_write,
+
+       .freeze                 = ahci_freeze,
+       .thaw                   = ahci_thaw,
+
+       .error_handler          = ahci_vt8251_error_handler,
+       .post_internal_cmd      = ahci_post_internal_cmd,
+
+       .pmp_attach             = ahci_pmp_attach,
+       .pmp_detach             = ahci_pmp_detach,
+
+#ifdef CONFIG_PM
        .port_suspend           = ahci_port_suspend,
        .port_resume            = ahci_port_resume,
+#endif
 
        .port_start             = ahci_port_start,
        .port_stop              = ahci_port_stop,
 };
 
+#define AHCI_HFLAGS(flags)     .private_data   = (void *)(flags)
+
 static const struct ata_port_info ahci_port_info[] = {
        /* board_ahci */
        {
-               .sht            = &ahci_sht,
-               .flags          = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
-                                 ATA_FLAG_MMIO | ATA_FLAG_PIO_DMA |
-                                 ATA_FLAG_SKIP_D2H_BSY,
+               .flags          = AHCI_FLAG_COMMON,
+               .link_flags     = AHCI_LFLAG_COMMON,
                .pio_mask       = 0x1f, /* pio0-4 */
-               .udma_mask      = 0x7f, /* udma0-6 ; FIXME */
+               .udma_mask      = ATA_UDMA6,
                .port_ops       = &ahci_ops,
        },
        /* board_ahci_vt8251 */
        {
-               .sht            = &ahci_sht,
-               .flags          = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
-                                 ATA_FLAG_MMIO | ATA_FLAG_PIO_DMA |
-                                 ATA_FLAG_SKIP_D2H_BSY | AHCI_FLAG_NO_NCQ,
+               AHCI_HFLAGS     (AHCI_HFLAG_NO_NCQ | AHCI_HFLAG_NO_PMP),
+               .flags          = AHCI_FLAG_COMMON,
+               .link_flags     = AHCI_LFLAG_COMMON | ATA_LFLAG_HRST_TO_RESUME,
                .pio_mask       = 0x1f, /* pio0-4 */
-               .udma_mask      = 0x7f, /* udma0-6 ; FIXME */
-               .port_ops       = &ahci_ops,
+               .udma_mask      = ATA_UDMA6,
+               .port_ops       = &ahci_vt8251_ops,
        },
        /* board_ahci_ign_iferr */
        {
-               .sht            = &ahci_sht,
+               AHCI_HFLAGS     (AHCI_HFLAG_IGN_IRQ_IF_ERR),
+               .flags          = AHCI_FLAG_COMMON,
+               .link_flags     = AHCI_LFLAG_COMMON,
+               .pio_mask       = 0x1f, /* pio0-4 */
+               .udma_mask      = ATA_UDMA6,
+               .port_ops       = &ahci_ops,
+       },
+       /* board_ahci_sb600 */
+       {
+               AHCI_HFLAGS     (AHCI_HFLAG_IGN_SERR_INTERNAL |
+                                AHCI_HFLAG_32BIT_ONLY | AHCI_HFLAG_NO_PMP),
+               .flags          = AHCI_FLAG_COMMON,
+               .link_flags     = AHCI_LFLAG_COMMON,
+               .pio_mask       = 0x1f, /* pio0-4 */
+               .udma_mask      = ATA_UDMA6,
+               .port_ops       = &ahci_ops,
+       },
+       /* board_ahci_mv */
+       {
+               AHCI_HFLAGS     (AHCI_HFLAG_NO_NCQ | AHCI_HFLAG_NO_MSI |
+                                AHCI_HFLAG_MV_PATA),
                .flags          = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
-                                 ATA_FLAG_MMIO | ATA_FLAG_PIO_DMA |
-                                 ATA_FLAG_SKIP_D2H_BSY |
-                                 AHCI_FLAG_IGN_IRQ_IF_ERR,
+                                 ATA_FLAG_MMIO | ATA_FLAG_PIO_DMA,
+               .link_flags     = AHCI_LFLAG_COMMON,
                .pio_mask       = 0x1f, /* pio0-4 */
-               .udma_mask      = 0x7f, /* udma0-6 ; FIXME */
+               .udma_mask      = ATA_UDMA6,
                .port_ops       = &ahci_ops,
        },
 };
@@ -315,7 +398,7 @@ static const struct pci_device_id ahci_pci_tbl[] = {
        { PCI_VDEVICE(INTEL, 0x27c1), board_ahci }, /* ICH7 */
        { PCI_VDEVICE(INTEL, 0x27c5), board_ahci }, /* ICH7M */
        { PCI_VDEVICE(INTEL, 0x27c3), board_ahci }, /* ICH7R */
-       { PCI_VDEVICE(AL, 0x5288), board_ahci }, /* ULi M5288 */
+       { PCI_VDEVICE(AL, 0x5288), board_ahci_ign_iferr }, /* ULi M5288 */
        { PCI_VDEVICE(INTEL, 0x2681), board_ahci }, /* ESB2 */
        { PCI_VDEVICE(INTEL, 0x2682), board_ahci }, /* ESB2 */
        { PCI_VDEVICE(INTEL, 0x2683), board_ahci }, /* ESB2 */
@@ -333,29 +416,43 @@ static const struct pci_device_id ahci_pci_tbl[] = {
        { PCI_VDEVICE(INTEL, 0x2929), board_ahci }, /* ICH9M */
        { PCI_VDEVICE(INTEL, 0x292a), board_ahci }, /* ICH9M */
        { PCI_VDEVICE(INTEL, 0x292b), board_ahci }, /* ICH9M */
+       { PCI_VDEVICE(INTEL, 0x292c), board_ahci }, /* ICH9M */
        { PCI_VDEVICE(INTEL, 0x292f), board_ahci }, /* ICH9M */
        { PCI_VDEVICE(INTEL, 0x294d), board_ahci }, /* ICH9 */
        { PCI_VDEVICE(INTEL, 0x294e), board_ahci }, /* ICH9M */
+       { PCI_VDEVICE(INTEL, 0x502a), board_ahci }, /* Tolapai */
+       { PCI_VDEVICE(INTEL, 0x502b), board_ahci }, /* Tolapai */
 
-       /* JMicron */
-       { PCI_VDEVICE(JMICRON, 0x2360), board_ahci_ign_iferr }, /* JMB360 */
-       { PCI_VDEVICE(JMICRON, 0x2361), board_ahci_ign_iferr }, /* JMB361 */
-       { PCI_VDEVICE(JMICRON, 0x2363), board_ahci_ign_iferr }, /* JMB363 */
-       { PCI_VDEVICE(JMICRON, 0x2365), board_ahci_ign_iferr }, /* JMB365 */
-       { PCI_VDEVICE(JMICRON, 0x2366), board_ahci_ign_iferr }, /* JMB366 */
+       /* JMicron 360/1/3/5/6, match class to avoid IDE function */
+       { PCI_VENDOR_ID_JMICRON, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID,
+         PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff, board_ahci_ign_iferr },
 
        /* ATI */
-       { PCI_VDEVICE(ATI, 0x4380), board_ahci }, /* ATI SB600 non-raid */
-       { PCI_VDEVICE(ATI, 0x4381), board_ahci }, /* ATI SB600 raid */
+       { PCI_VDEVICE(ATI, 0x4380), board_ahci_sb600 }, /* ATI SB600 */
+       { PCI_VDEVICE(ATI, 0x4390), board_ahci_sb600 }, /* ATI SB700/800 */
+       { PCI_VDEVICE(ATI, 0x4391), board_ahci_sb600 }, /* ATI SB700/800 */
+       { PCI_VDEVICE(ATI, 0x4392), board_ahci_sb600 }, /* ATI SB700/800 */
+       { PCI_VDEVICE(ATI, 0x4393), board_ahci_sb600 }, /* ATI SB700/800 */
+       { PCI_VDEVICE(ATI, 0x4394), board_ahci_sb600 }, /* ATI SB700/800 */
+       { PCI_VDEVICE(ATI, 0x4395), board_ahci_sb600 }, /* ATI SB700/800 */
 
        /* VIA */
        { PCI_VDEVICE(VIA, 0x3349), board_ahci_vt8251 }, /* VIA VT8251 */
+       { PCI_VDEVICE(VIA, 0x6287), board_ahci_vt8251 }, /* VIA VT8251 */
 
        /* NVIDIA */
        { PCI_VDEVICE(NVIDIA, 0x044c), board_ahci },            /* MCP65 */
        { PCI_VDEVICE(NVIDIA, 0x044d), board_ahci },            /* MCP65 */
        { PCI_VDEVICE(NVIDIA, 0x044e), board_ahci },            /* MCP65 */
        { PCI_VDEVICE(NVIDIA, 0x044f), board_ahci },            /* MCP65 */
+       { PCI_VDEVICE(NVIDIA, 0x045c), board_ahci },            /* MCP65 */
+       { PCI_VDEVICE(NVIDIA, 0x045d), board_ahci },            /* MCP65 */
+       { PCI_VDEVICE(NVIDIA, 0x045e), board_ahci },            /* MCP65 */
+       { PCI_VDEVICE(NVIDIA, 0x045f), board_ahci },            /* MCP65 */
+       { PCI_VDEVICE(NVIDIA, 0x0550), board_ahci },            /* MCP67 */
+       { PCI_VDEVICE(NVIDIA, 0x0551), board_ahci },            /* MCP67 */
+       { PCI_VDEVICE(NVIDIA, 0x0552), board_ahci },            /* MCP67 */
+       { PCI_VDEVICE(NVIDIA, 0x0553), board_ahci },            /* MCP67 */
        { PCI_VDEVICE(NVIDIA, 0x0554), board_ahci },            /* MCP67 */
        { PCI_VDEVICE(NVIDIA, 0x0555), board_ahci },            /* MCP67 */
        { PCI_VDEVICE(NVIDIA, 0x0556), board_ahci },            /* MCP67 */
@@ -364,12 +461,51 @@ static const struct pci_device_id ahci_pci_tbl[] = {
        { PCI_VDEVICE(NVIDIA, 0x0559), board_ahci },            /* MCP67 */
        { PCI_VDEVICE(NVIDIA, 0x055a), board_ahci },            /* MCP67 */
        { PCI_VDEVICE(NVIDIA, 0x055b), board_ahci },            /* MCP67 */
+       { PCI_VDEVICE(NVIDIA, 0x07f0), board_ahci },            /* MCP73 */
+       { PCI_VDEVICE(NVIDIA, 0x07f1), board_ahci },            /* MCP73 */
+       { PCI_VDEVICE(NVIDIA, 0x07f2), board_ahci },            /* MCP73 */
+       { PCI_VDEVICE(NVIDIA, 0x07f3), board_ahci },            /* MCP73 */
+       { PCI_VDEVICE(NVIDIA, 0x07f4), board_ahci },            /* MCP73 */
+       { PCI_VDEVICE(NVIDIA, 0x07f5), board_ahci },            /* MCP73 */
+       { PCI_VDEVICE(NVIDIA, 0x07f6), board_ahci },            /* MCP73 */
+       { PCI_VDEVICE(NVIDIA, 0x07f7), board_ahci },            /* MCP73 */
+       { PCI_VDEVICE(NVIDIA, 0x07f8), board_ahci },            /* MCP73 */
+       { PCI_VDEVICE(NVIDIA, 0x07f9), board_ahci },            /* MCP73 */
+       { PCI_VDEVICE(NVIDIA, 0x07fa), board_ahci },            /* MCP73 */
+       { PCI_VDEVICE(NVIDIA, 0x07fb), board_ahci },            /* MCP73 */
+       { PCI_VDEVICE(NVIDIA, 0x0ad0), board_ahci },            /* MCP77 */
+       { PCI_VDEVICE(NVIDIA, 0x0ad1), board_ahci },            /* MCP77 */
+       { PCI_VDEVICE(NVIDIA, 0x0ad2), board_ahci },            /* MCP77 */
+       { PCI_VDEVICE(NVIDIA, 0x0ad3), board_ahci },            /* MCP77 */
+       { PCI_VDEVICE(NVIDIA, 0x0ad4), board_ahci },            /* MCP77 */
+       { PCI_VDEVICE(NVIDIA, 0x0ad5), board_ahci },            /* MCP77 */
+       { PCI_VDEVICE(NVIDIA, 0x0ad6), board_ahci },            /* MCP77 */
+       { PCI_VDEVICE(NVIDIA, 0x0ad7), board_ahci },            /* MCP77 */
+       { PCI_VDEVICE(NVIDIA, 0x0ad8), board_ahci },            /* MCP77 */
+       { PCI_VDEVICE(NVIDIA, 0x0ad9), board_ahci },            /* MCP77 */
+       { PCI_VDEVICE(NVIDIA, 0x0ada), board_ahci },            /* MCP77 */
+       { PCI_VDEVICE(NVIDIA, 0x0adb), board_ahci },            /* MCP77 */
+       { PCI_VDEVICE(NVIDIA, 0x0ab8), board_ahci },            /* MCP79 */
+       { PCI_VDEVICE(NVIDIA, 0x0ab9), board_ahci },            /* MCP79 */
+       { PCI_VDEVICE(NVIDIA, 0x0aba), board_ahci },            /* MCP79 */
+       { PCI_VDEVICE(NVIDIA, 0x0abb), board_ahci },            /* MCP79 */
+       { PCI_VDEVICE(NVIDIA, 0x0abc), board_ahci },            /* MCP79 */
+       { PCI_VDEVICE(NVIDIA, 0x0abd), board_ahci },            /* MCP79 */
+       { PCI_VDEVICE(NVIDIA, 0x0abe), board_ahci },            /* MCP79 */
+       { PCI_VDEVICE(NVIDIA, 0x0abf), board_ahci },            /* MCP79 */
 
        /* SiS */
        { PCI_VDEVICE(SI, 0x1184), board_ahci }, /* SiS 966 */
        { PCI_VDEVICE(SI, 0x1185), board_ahci }, /* SiS 966 */
        { PCI_VDEVICE(SI, 0x0186), board_ahci }, /* SiS 968 */
 
+       /* Marvell */
+       { PCI_VDEVICE(MARVELL, 0x6145), board_ahci_mv },        /* 6145 */
+
+       /* Generic, PCI class code for AHCI */
+       { PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID,
+         PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff, board_ahci },
+
        { }     /* terminate list */
 };
 
@@ -378,58 +514,195 @@ static struct pci_driver ahci_pci_driver = {
        .name                   = DRV_NAME,
        .id_table               = ahci_pci_tbl,
        .probe                  = ahci_init_one,
+       .remove                 = ata_pci_remove_one,
+#ifdef CONFIG_PM
        .suspend                = ahci_pci_device_suspend,
        .resume                 = ahci_pci_device_resume,
-       .remove                 = ahci_remove_one,
+#endif
 };
 
 
-static inline unsigned long ahci_port_base_ul (unsigned long base, unsigned int port)
+static inline int ahci_nr_ports(u32 cap)
+{
+       return (cap & 0x1f) + 1;
+}
+
+static inline void __iomem *__ahci_port_base(struct ata_host *host,
+                                            unsigned int port_no)
 {
-       return base + 0x100 + (port * 0x80);
+       void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
+
+       return mmio + 0x100 + (port_no * 0x80);
 }
 
-static inline void __iomem *ahci_port_base (void __iomem *base, unsigned int port)
+static inline void __iomem *ahci_port_base(struct ata_port *ap)
 {
-       return (void __iomem *) ahci_port_base_ul((unsigned long)base, port);
+       return __ahci_port_base(ap->host, ap->port_no);
 }
 
-static u32 ahci_scr_read (struct ata_port *ap, unsigned int sc_reg_in)
+/**
+ *     ahci_save_initial_config - Save and fixup initial config values
+ *     @pdev: target PCI device
+ *     @hpriv: host private area to store config values
+ *
+ *     Some registers containing configuration info might be setup by
+ *     BIOS and might be cleared on reset.  This function saves the
+ *     initial values of those registers into @hpriv such that they
+ *     can be restored after controller reset.
+ *
+ *     If inconsistent, config values are fixed up by this function.
+ *
+ *     LOCKING:
+ *     None.
+ */
+static void ahci_save_initial_config(struct pci_dev *pdev,
+                                    struct ahci_host_priv *hpriv)
 {
-       unsigned int sc_reg;
-
-       switch (sc_reg_in) {
-       case SCR_STATUS:        sc_reg = 0; break;
-       case SCR_CONTROL:       sc_reg = 1; break;
-       case SCR_ERROR:         sc_reg = 2; break;
-       case SCR_ACTIVE:        sc_reg = 3; break;
-       default:
-               return 0xffffffffU;
+       void __iomem *mmio = pcim_iomap_table(pdev)[AHCI_PCI_BAR];
+       u32 cap, port_map;
+       int i;
+
+       /* Values prefixed with saved_ are written back to host after
+        * reset.  Values without are used for driver operation.
+        */
+       hpriv->saved_cap = cap = readl(mmio + HOST_CAP);
+       hpriv->saved_port_map = port_map = readl(mmio + HOST_PORTS_IMPL);
+
+       /* some chips have errata preventing 64bit use */
+       if ((cap & HOST_CAP_64) && (hpriv->flags & AHCI_HFLAG_32BIT_ONLY)) {
+               dev_printk(KERN_INFO, &pdev->dev,
+                          "controller can't do 64bit DMA, forcing 32bit\n");
+               cap &= ~HOST_CAP_64;
+       }
+
+       if ((cap & HOST_CAP_NCQ) && (hpriv->flags & AHCI_HFLAG_NO_NCQ)) {
+               dev_printk(KERN_INFO, &pdev->dev,
+                          "controller can't do NCQ, turning off CAP_NCQ\n");
+               cap &= ~HOST_CAP_NCQ;
+       }
+
+       if ((cap && HOST_CAP_PMP) && (hpriv->flags & AHCI_HFLAG_NO_PMP)) {
+               dev_printk(KERN_INFO, &pdev->dev,
+                          "controller can't do PMP, turning off CAP_PMP\n");
+               cap &= ~HOST_CAP_PMP;
+       }
+
+       /*
+        * Temporary Marvell 6145 hack: PATA port presence
+        * is asserted through the standard AHCI port
+        * presence register, as bit 4 (counting from 0)
+        */
+       if (hpriv->flags & AHCI_HFLAG_MV_PATA) {
+               dev_printk(KERN_ERR, &pdev->dev,
+                          "MV_AHCI HACK: port_map %x -> %x\n",
+                          hpriv->port_map,
+                          hpriv->port_map & 0xf);
+
+               port_map &= 0xf;
+       }
+
+       /* cross check port_map and cap.n_ports */
+       if (port_map) {
+               u32 tmp_port_map = port_map;
+               int n_ports = ahci_nr_ports(cap);
+
+               for (i = 0; i < AHCI_MAX_PORTS && n_ports; i++) {
+                       if (tmp_port_map & (1 << i)) {
+                               n_ports--;
+                               tmp_port_map &= ~(1 << i);
+                       }
+               }
+
+               /* If n_ports and port_map are inconsistent, whine and
+                * clear port_map and let it be generated from n_ports.
+                */
+               if (n_ports || tmp_port_map) {
+                       dev_printk(KERN_WARNING, &pdev->dev,
+                                  "nr_ports (%u) and implemented port map "
+                                  "(0x%x) don't match, using nr_ports\n",
+                                  ahci_nr_ports(cap), port_map);
+                       port_map = 0;
+               }
+       }
+
+       /* fabricate port_map from cap.nr_ports */
+       if (!port_map) {
+               port_map = (1 << ahci_nr_ports(cap)) - 1;
+               dev_printk(KERN_WARNING, &pdev->dev,
+                          "forcing PORTS_IMPL to 0x%x\n", port_map);
+
+               /* write the fixed up value to the PI register */
+               hpriv->saved_port_map = port_map;
        }
 
-       return readl((void __iomem *) ap->ioaddr.scr_addr + (sc_reg * 4));
+       /* record values to use during operation */
+       hpriv->cap = cap;
+       hpriv->port_map = port_map;
 }
 
+/**
+ *     ahci_restore_initial_config - Restore initial config
+ *     @host: target ATA host
+ *
+ *     Restore initial config stored by ahci_save_initial_config().
+ *
+ *     LOCKING:
+ *     None.
+ */
+static void ahci_restore_initial_config(struct ata_host *host)
+{
+       struct ahci_host_priv *hpriv = host->private_data;
+       void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
+
+       writel(hpriv->saved_cap, mmio + HOST_CAP);
+       writel(hpriv->saved_port_map, mmio + HOST_PORTS_IMPL);
+       (void) readl(mmio + HOST_PORTS_IMPL);   /* flush */
+}
 
-static void ahci_scr_write (struct ata_port *ap, unsigned int sc_reg_in,
-                              u32 val)
+static unsigned ahci_scr_offset(struct ata_port *ap, unsigned int sc_reg)
 {
-       unsigned int sc_reg;
-
-       switch (sc_reg_in) {
-       case SCR_STATUS:        sc_reg = 0; break;
-       case SCR_CONTROL:       sc_reg = 1; break;
-       case SCR_ERROR:         sc_reg = 2; break;
-       case SCR_ACTIVE:        sc_reg = 3; break;
-       default:
-               return;
+       static const int offset[] = {
+               [SCR_STATUS]            = PORT_SCR_STAT,
+               [SCR_CONTROL]           = PORT_SCR_CTL,
+               [SCR_ERROR]             = PORT_SCR_ERR,
+               [SCR_ACTIVE]            = PORT_SCR_ACT,
+               [SCR_NOTIFICATION]      = PORT_SCR_NTF,
+       };
+       struct ahci_host_priv *hpriv = ap->host->private_data;
+
+       if (sc_reg < ARRAY_SIZE(offset) &&
+           (sc_reg != SCR_NOTIFICATION || (hpriv->cap & HOST_CAP_SNTF)))
+               return offset[sc_reg];
+       return 0;
+}
+
+static int ahci_scr_read(struct ata_port *ap, unsigned int sc_reg, u32 *val)
+{
+       void __iomem *port_mmio = ahci_port_base(ap);
+       int offset = ahci_scr_offset(ap, sc_reg);
+
+       if (offset) {
+               *val = readl(port_mmio + offset);
+               return 0;
        }
+       return -EINVAL;
+}
+
+static int ahci_scr_write(struct ata_port *ap, unsigned int sc_reg, u32 val)
+{
+       void __iomem *port_mmio = ahci_port_base(ap);
+       int offset = ahci_scr_offset(ap, sc_reg);
 
-       writel(val, (void __iomem *) ap->ioaddr.scr_addr + (sc_reg * 4));
+       if (offset) {
+               writel(val, port_mmio + offset);
+               return 0;
+       }
+       return -EINVAL;
 }
 
-static void ahci_start_engine(void __iomem *port_mmio)
+static void ahci_start_engine(struct ata_port *ap)
 {
+       void __iomem *port_mmio = ahci_port_base(ap);
        u32 tmp;
 
        /* start DMA */
@@ -439,8 +712,9 @@ static void ahci_start_engine(void __iomem *port_mmio)
        readl(port_mmio + PORT_CMD); /* flush */
 }
 
-static int ahci_stop_engine(void __iomem *port_mmio)
+static int ahci_stop_engine(struct ata_port *ap)
 {
+       void __iomem *port_mmio = ahci_port_base(ap);
        u32 tmp;
 
        tmp = readl(port_mmio + PORT_CMD);
@@ -462,19 +736,23 @@ static int ahci_stop_engine(void __iomem *port_mmio)
        return 0;
 }
 
-static void ahci_start_fis_rx(void __iomem *port_mmio, u32 cap,
-                             dma_addr_t cmd_slot_dma, dma_addr_t rx_fis_dma)
+static void ahci_start_fis_rx(struct ata_port *ap)
 {
+       void __iomem *port_mmio = ahci_port_base(ap);
+       struct ahci_host_priv *hpriv = ap->host->private_data;
+       struct ahci_port_priv *pp = ap->private_data;
        u32 tmp;
 
        /* set FIS registers */
-       if (cap & HOST_CAP_64)
-               writel((cmd_slot_dma >> 16) >> 16, port_mmio + PORT_LST_ADDR_HI);
-       writel(cmd_slot_dma & 0xffffffff, port_mmio + PORT_LST_ADDR);
+       if (hpriv->cap & HOST_CAP_64)
+               writel((pp->cmd_slot_dma >> 16) >> 16,
+                      port_mmio + PORT_LST_ADDR_HI);
+       writel(pp->cmd_slot_dma & 0xffffffff, port_mmio + PORT_LST_ADDR);
 
-       if (cap & HOST_CAP_64)
-               writel((rx_fis_dma >> 16) >> 16, port_mmio + PORT_FIS_ADDR_HI);
-       writel(rx_fis_dma & 0xffffffff, port_mmio + PORT_FIS_ADDR);
+       if (hpriv->cap & HOST_CAP_64)
+               writel((pp->rx_fis_dma >> 16) >> 16,
+                      port_mmio + PORT_FIS_ADDR_HI);
+       writel(pp->rx_fis_dma & 0xffffffff, port_mmio + PORT_FIS_ADDR);
 
        /* enable FIS reception */
        tmp = readl(port_mmio + PORT_CMD);
@@ -485,8 +763,9 @@ static void ahci_start_fis_rx(void __iomem *port_mmio, u32 cap,
        readl(port_mmio + PORT_CMD);
 }
 
-static int ahci_stop_fis_rx(void __iomem *port_mmio)
+static int ahci_stop_fis_rx(struct ata_port *ap)
 {
+       void __iomem *port_mmio = ahci_port_base(ap);
        u32 tmp;
 
        /* disable FIS reception */
@@ -503,14 +782,16 @@ static int ahci_stop_fis_rx(void __iomem *port_mmio)
        return 0;
 }
 
-static void ahci_power_up(void __iomem *port_mmio, u32 cap)
+static void ahci_power_up(struct ata_port *ap)
 {
+       struct ahci_host_priv *hpriv = ap->host->private_data;
+       void __iomem *port_mmio = ahci_port_base(ap);
        u32 cmd;
 
        cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;
 
        /* spin up device */
-       if (cap & HOST_CAP_SSS) {
+       if (hpriv->cap & HOST_CAP_SSS) {
                cmd |= PORT_CMD_SPIN_UP;
                writel(cmd, port_mmio + PORT_CMD);
        }
@@ -519,88 +800,72 @@ static void ahci_power_up(void __iomem *port_mmio, u32 cap)
        writel(cmd | PORT_CMD_ICC_ACTIVE, port_mmio + PORT_CMD);
 }
 
-static void ahci_power_down(void __iomem *port_mmio, u32 cap)
+#ifdef CONFIG_PM
+static void ahci_power_down(struct ata_port *ap)
 {
+       struct ahci_host_priv *hpriv = ap->host->private_data;
+       void __iomem *port_mmio = ahci_port_base(ap);
        u32 cmd, scontrol;
 
-       cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;
-
-       if (cap & HOST_CAP_SSC) {
-               /* enable transitions to slumber mode */
-               scontrol = readl(port_mmio + PORT_SCR_CTL);
-               if ((scontrol & 0x0f00) > 0x100) {
-                       scontrol &= ~0xf00;
-                       writel(scontrol, port_mmio + PORT_SCR_CTL);
-               }
-
-               /* put device into slumber mode */
-               writel(cmd | PORT_CMD_ICC_SLUMBER, port_mmio + PORT_CMD);
-
-               /* wait for the transition to complete */
-               ata_wait_register(port_mmio + PORT_CMD, PORT_CMD_ICC_SLUMBER,
-                                 PORT_CMD_ICC_SLUMBER, 1, 50);
-       }
+       if (!(hpriv->cap & HOST_CAP_SSS))
+               return;
 
-       /* put device into listen mode */
-       if (cap & HOST_CAP_SSS) {
-               /* first set PxSCTL.DET to 0 */
-               scontrol = readl(port_mmio + PORT_SCR_CTL);
-               scontrol &= ~0xf;
-               writel(scontrol, port_mmio + PORT_SCR_CTL);
+       /* put device into listen mode, first set PxSCTL.DET to 0 */
+       scontrol = readl(port_mmio + PORT_SCR_CTL);
+       scontrol &= ~0xf;
+       writel(scontrol, port_mmio + PORT_SCR_CTL);
 
-               /* then set PxCMD.SUD to 0 */
-               cmd &= ~PORT_CMD_SPIN_UP;
-               writel(cmd, port_mmio + PORT_CMD);
-       }
+       /* then set PxCMD.SUD to 0 */
+       cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK;
+       cmd &= ~PORT_CMD_SPIN_UP;
+       writel(cmd, port_mmio + PORT_CMD);
 }
+#endif
 
-static void ahci_init_port(void __iomem *port_mmio, u32 cap,
-                          dma_addr_t cmd_slot_dma, dma_addr_t rx_fis_dma)
+static void ahci_start_port(struct ata_port *ap)
 {
-       /* power up */
-       ahci_power_up(port_mmio, cap);
-
        /* enable FIS reception */
-       ahci_start_fis_rx(port_mmio, cap, cmd_slot_dma, rx_fis_dma);
+       ahci_start_fis_rx(ap);
 
        /* enable DMA */
-       ahci_start_engine(port_mmio);
+       ahci_start_engine(ap);
 }
 
-static int ahci_deinit_port(void __iomem *port_mmio, u32 cap, const char **emsg)
+static int ahci_deinit_port(struct ata_port *ap, const char **emsg)
 {
        int rc;
 
        /* disable DMA */
-       rc = ahci_stop_engine(port_mmio);
+       rc = ahci_stop_engine(ap);
        if (rc) {
                *emsg = "failed to stop engine";
                return rc;
        }
 
        /* disable FIS reception */
-       rc = ahci_stop_fis_rx(port_mmio);
+       rc = ahci_stop_fis_rx(ap);
        if (rc) {
                *emsg = "failed stop FIS RX";
                return rc;
        }
 
-       /* put device into slumber mode */
-       ahci_power_down(port_mmio, cap);
-
        return 0;
 }
 
-static int ahci_reset_controller(void __iomem *mmio, struct pci_dev *pdev)
+static int ahci_reset_controller(struct ata_host *host)
 {
-       u32 cap_save, tmp;
+       struct pci_dev *pdev = to_pci_dev(host->dev);
+       void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
+       u32 tmp;
 
-       cap_save = readl(mmio + HOST_CAP);
-       cap_save &= ( (1<<28) | (1<<17) );
-       cap_save |= (1 << 27);
+       /* we must be in AHCI mode, before using anything
+        * AHCI-specific, such as HOST_RESET.
+        */
+       tmp = readl(mmio + HOST_CTL);
+       if (!(tmp & HOST_AHCI_EN))
+               writel(tmp | HOST_AHCI_EN, mmio + HOST_CTL);
 
        /* global controller reset */
-       tmp = readl(mmio + HOST_CTL);
        if ((tmp & HOST_RESET) == 0) {
                writel(tmp | HOST_RESET, mmio + HOST_CTL);
                readl(mmio + HOST_CTL); /* flush */
@@ -613,16 +878,17 @@ static int ahci_reset_controller(void __iomem *mmio, struct pci_dev *pdev)
 
        tmp = readl(mmio + HOST_CTL);
        if (tmp & HOST_RESET) {
-               dev_printk(KERN_ERR, &pdev->dev,
+               dev_printk(KERN_ERR, host->dev,
                           "controller reset failed (0x%x)\n", tmp);
                return -EIO;
        }
 
+       /* turn on AHCI mode */
        writel(HOST_AHCI_EN, mmio + HOST_CTL);
        (void) readl(mmio + HOST_CTL);  /* flush */
-       writel(cap_save, mmio + HOST_CAP);
-       writel(0xf, mmio + HOST_PORTS_IMPL);
-       (void) readl(mmio + HOST_PORTS_IMPL);   /* flush */
+
+       /* some registers might be cleared on reset.  restore initial values */
+       ahci_restore_initial_config(host);
 
        if (pdev->vendor == PCI_VENDOR_ID_INTEL) {
                u16 tmp16;
@@ -636,39 +902,63 @@ static int ahci_reset_controller(void __iomem *mmio, struct pci_dev *pdev)
        return 0;
 }
 
-static void ahci_init_controller(void __iomem *mmio, struct pci_dev *pdev,
-                                int n_ports, u32 cap)
+static void ahci_port_init(struct pci_dev *pdev, struct ata_port *ap,
+                          int port_no, void __iomem *mmio,
+                          void __iomem *port_mmio)
 {
-       int i, rc;
+       const char *emsg = NULL;
+       int rc;
        u32 tmp;
 
-       for (i = 0; i < n_ports; i++) {
-               void __iomem *port_mmio = ahci_port_base(mmio, i);
-               const char *emsg = NULL;
+       /* make sure port is not active */
+       rc = ahci_deinit_port(ap, &emsg);
+       if (rc)
+               dev_printk(KERN_WARNING, &pdev->dev,
+                          "%s (%d)\n", emsg, rc);
 
-#if 0 /* BIOSen initialize this incorrectly */
-               if (!(hpriv->port_map & (1 << i)))
-                       continue;
-#endif
+       /* clear SError */
+       tmp = readl(port_mmio + PORT_SCR_ERR);
+       VPRINTK("PORT_SCR_ERR 0x%x\n", tmp);
+       writel(tmp, port_mmio + PORT_SCR_ERR);
 
-               /* make sure port is not active */
-               rc = ahci_deinit_port(port_mmio, cap, &emsg);
-               if (rc)
-                       dev_printk(KERN_WARNING, &pdev->dev,
-                                  "%s (%d)\n", emsg, rc);
+       /* clear port IRQ */
+       tmp = readl(port_mmio + PORT_IRQ_STAT);
+       VPRINTK("PORT_IRQ_STAT 0x%x\n", tmp);
+       if (tmp)
+               writel(tmp, port_mmio + PORT_IRQ_STAT);
+
+       writel(1 << port_no, mmio + HOST_IRQ_STAT);
+}
+
+static void ahci_init_controller(struct ata_host *host)
+{
+       struct ahci_host_priv *hpriv = host->private_data;
+       struct pci_dev *pdev = to_pci_dev(host->dev);
+       void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
+       int i;
+       void __iomem *port_mmio;
+       u32 tmp;
 
-               /* clear SError */
-               tmp = readl(port_mmio + PORT_SCR_ERR);
-               VPRINTK("PORT_SCR_ERR 0x%x\n", tmp);
-               writel(tmp, port_mmio + PORT_SCR_ERR);
+       if (hpriv->flags & AHCI_HFLAG_MV_PATA) {
+               port_mmio = __ahci_port_base(host, 4);
+
+               writel(0, port_mmio + PORT_IRQ_MASK);
 
                /* clear port IRQ */
                tmp = readl(port_mmio + PORT_IRQ_STAT);
                VPRINTK("PORT_IRQ_STAT 0x%x\n", tmp);
                if (tmp)
                        writel(tmp, port_mmio + PORT_IRQ_STAT);
+       }
+
+       for (i = 0; i < host->n_ports; i++) {
+               struct ata_port *ap = host->ports[i];
+
+               port_mmio = ahci_port_base(ap);
+               if (ata_port_is_dummy(ap))
+                       continue;
 
-               writel(1 << i, mmio + HOST_IRQ_STAT);
+               ahci_port_init(pdev, ap, i, mmio, port_mmio);
        }
 
        tmp = readl(mmio + HOST_CTL);
@@ -680,7 +970,7 @@ static void ahci_init_controller(void __iomem *mmio, struct pci_dev *pdev,
 
 static unsigned int ahci_dev_classify(struct ata_port *ap)
 {
-       void __iomem *port_mmio = (void __iomem *) ap->ioaddr.cmd_addr;
+       void __iomem *port_mmio = ahci_port_base(ap);
        struct ata_taskfile tf;
        u32 tmp;
 
@@ -706,85 +996,115 @@ static void ahci_fill_cmd_slot(struct ahci_port_priv *pp, unsigned int tag,
        pp->cmd_slot[tag].tbl_addr_hi = cpu_to_le32((cmd_tbl_dma >> 16) >> 16);
 }
 
-static int ahci_clo(struct ata_port *ap)
+static int ahci_kick_engine(struct ata_port *ap, int force_restart)
 {
-       void __iomem *port_mmio = (void __iomem *) ap->ioaddr.cmd_addr;
+       void __iomem *port_mmio = ap->ioaddr.cmd_addr;
        struct ahci_host_priv *hpriv = ap->host->private_data;
        u32 tmp;
+       int busy, rc;
+
+       /* do we need to kick the port? */
+       busy = ahci_check_status(ap) & (ATA_BUSY | ATA_DRQ);
+       if (!busy && !force_restart)
+               return 0;
+
+       /* stop engine */
+       rc = ahci_stop_engine(ap);
+       if (rc)
+               goto out_restart;
+
+       /* need to do CLO? */
+       if (!busy) {
+               rc = 0;
+               goto out_restart;
+       }
 
-       if (!(hpriv->cap & HOST_CAP_CLO))
-               return -EOPNOTSUPP;
+       if (!(hpriv->cap & HOST_CAP_CLO)) {
+               rc = -EOPNOTSUPP;
+               goto out_restart;
+       }
 
+       /* perform CLO */
        tmp = readl(port_mmio + PORT_CMD);
        tmp |= PORT_CMD_CLO;
        writel(tmp, port_mmio + PORT_CMD);
 
+       rc = 0;
        tmp = ata_wait_register(port_mmio + PORT_CMD,
                                PORT_CMD_CLO, PORT_CMD_CLO, 1, 500);
        if (tmp & PORT_CMD_CLO)
-               return -EIO;
+               rc = -EIO;
 
-       return 0;
+       /* restart engine */
+ out_restart:
+       ahci_start_engine(ap);
+       return rc;
 }
 
-static int ahci_softreset(struct ata_port *ap, unsigned int *class)
+static int ahci_exec_polled_cmd(struct ata_port *ap, int pmp,
+                               struct ata_taskfile *tf, int is_cmd, u16 flags,
+                               unsigned long timeout_msec)
 {
-       struct ahci_port_priv *pp = ap->private_data;
-       void __iomem *mmio = ap->host->mmio_base;
-       void __iomem *port_mmio = ahci_port_base(mmio, ap->port_no);
        const u32 cmd_fis_len = 5; /* five dwords */
+       struct ahci_port_priv *pp = ap->private_data;
+       void __iomem *port_mmio = ahci_port_base(ap);
+       u8 *fis = pp->cmd_tbl;
+       u32 tmp;
+
+       /* prep the command */
+       ata_tf_to_fis(tf, pmp, is_cmd, fis);
+       ahci_fill_cmd_slot(pp, 0, cmd_fis_len | flags | (pmp << 12));
+
+       /* issue & wait */
+       writel(1, port_mmio + PORT_CMD_ISSUE);
+
+       if (timeout_msec) {
+               tmp = ata_wait_register(port_mmio + PORT_CMD_ISSUE, 0x1, 0x1,
+                                       1, timeout_msec);
+               if (tmp & 0x1) {
+                       ahci_kick_engine(ap, 1);
+                       return -EBUSY;
+               }
+       } else
+               readl(port_mmio + PORT_CMD_ISSUE);      /* flush */
+
+       return 0;
+}
+
+static int ahci_do_softreset(struct ata_link *link, unsigned int *class,
+                            int pmp, unsigned long deadline)
+{
+       struct ata_port *ap = link->ap;
        const char *reason = NULL;
+       unsigned long now, msecs;
        struct ata_taskfile tf;
-       u32 tmp;
-       u8 *fis;
        int rc;
 
        DPRINTK("ENTER\n");
 
-       if (ata_port_offline(ap)) {
+       if (ata_link_offline(link)) {
                DPRINTK("PHY reports no device\n");
                *class = ATA_DEV_NONE;
                return 0;
        }
 
        /* prepare for SRST (AHCI-1.1 10.4.1) */
-       rc = ahci_stop_engine(port_mmio);
-       if (rc) {
-               reason = "failed to stop engine";
-               goto fail_restart;
-       }
-
-       /* check BUSY/DRQ, perform Command List Override if necessary */
-       if (ahci_check_status(ap) & (ATA_BUSY | ATA_DRQ)) {
-               rc = ahci_clo(ap);
-
-               if (rc == -EOPNOTSUPP) {
-                       reason = "port busy but CLO unavailable";
-                       goto fail_restart;
-               } else if (rc) {
-                       reason = "port busy but CLO failed";
-                       goto fail_restart;
-               }
-       }
-
-       /* restart engine */
-       ahci_start_engine(port_mmio);
+       rc = ahci_kick_engine(ap, 1);
+       if (rc)
+               ata_link_printk(link, KERN_WARNING,
+                               "failed to reset engine (errno=%d)", rc);
 
-       ata_tf_init(ap->device, &tf);
-       fis = pp->cmd_tbl;
+       ata_tf_init(link->device, &tf);
 
        /* issue the first D2H Register FIS */
-       ahci_fill_cmd_slot(pp, 0,
-                          cmd_fis_len | AHCI_CMD_RESET | AHCI_CMD_CLR_BUSY);
+       msecs = 0;
+       now = jiffies;
+       if (time_after(now, deadline))
+               msecs = jiffies_to_msecs(deadline - now);
 
        tf.ctl |= ATA_SRST;
-       ata_tf_to_fis(&tf, fis, 0);
-       fis[1] &= ~(1 << 7);    /* turn off Command FIS bit */
-
-       writel(1, port_mmio + PORT_CMD_ISSUE);
-
-       tmp = ata_wait_register(port_mmio + PORT_CMD_ISSUE, 0x1, 0x1, 1, 500);
-       if (tmp & 0x1) {
+       if (ahci_exec_polled_cmd(ap, pmp, &tf, 0,
+                                AHCI_CMD_RESET | AHCI_CMD_CLR_BUSY, msecs)) {
                rc = -EIO;
                reason = "1st FIS failed";
                goto fail;
@@ -794,14 +1114,8 @@ static int ahci_softreset(struct ata_port *ap, unsigned int *class)
        msleep(1);
 
        /* issue the second D2H Register FIS */
-       ahci_fill_cmd_slot(pp, 0, cmd_fis_len);
-
        tf.ctl &= ~ATA_SRST;
-       ata_tf_to_fis(&tf, fis, 0);
-       fis[1] &= ~(1 << 7);    /* turn off Command FIS bit */
-
-       writel(1, port_mmio + PORT_CMD_ISSUE);
-       readl(port_mmio + PORT_CMD_ISSUE);      /* flush */
+       ahci_exec_polled_cmd(ap, pmp, &tf, 0, 0, 0);
 
        /* spec mandates ">= 2ms" before checking status.
         * We wait 150ms, because that was the magic delay used for
@@ -813,63 +1127,99 @@ static int ahci_softreset(struct ata_port *ap, unsigned int *class)
         */
        msleep(150);
 
-       *class = ATA_DEV_NONE;
-       if (ata_port_online(ap)) {
-               if (ata_busy_sleep(ap, ATA_TMOUT_BOOT_QUICK, ATA_TMOUT_BOOT)) {
-                       rc = -EIO;
-                       reason = "device not ready";
-                       goto fail;
-               }
-               *class = ahci_dev_classify(ap);
+       rc = ata_wait_ready(ap, deadline);
+       /* link occupied, -ENODEV too is an error */
+       if (rc) {
+               reason = "device not ready";
+               goto fail;
        }
+       *class = ahci_dev_classify(ap);
 
        DPRINTK("EXIT, class=%u\n", *class);
        return 0;
 
- fail_restart:
-       ahci_start_engine(port_mmio);
  fail:
-       ata_port_printk(ap, KERN_ERR, "softreset failed (%s)\n", reason);
+       ata_link_printk(link, KERN_ERR, "softreset failed (%s)\n", reason);
        return rc;
 }
 
-static int ahci_hardreset(struct ata_port *ap, unsigned int *class)
+static int ahci_softreset(struct ata_link *link, unsigned int *class,
+                         unsigned long deadline)
 {
+       int pmp = 0;
+
+       if (link->ap->flags & ATA_FLAG_PMP)
+               pmp = SATA_PMP_CTRL_PORT;
+
+       return ahci_do_softreset(link, class, pmp, deadline);
+}
+
+static int ahci_hardreset(struct ata_link *link, unsigned int *class,
+                         unsigned long deadline)
+{
+       struct ata_port *ap = link->ap;
        struct ahci_port_priv *pp = ap->private_data;
        u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG;
        struct ata_taskfile tf;
-       void __iomem *mmio = ap->host->mmio_base;
-       void __iomem *port_mmio = ahci_port_base(mmio, ap->port_no);
        int rc;
 
        DPRINTK("ENTER\n");
 
-       ahci_stop_engine(port_mmio);
+       ahci_stop_engine(ap);
 
        /* clear D2H reception area to properly wait for D2H FIS */
-       ata_tf_init(ap->device, &tf);
-       tf.command = 0xff;
-       ata_tf_to_fis(&tf, d2h_fis, 0);
+       ata_tf_init(link->device, &tf);
+       tf.command = 0x80;
+       ata_tf_to_fis(&tf, 0, 0, d2h_fis);
 
-       rc = sata_std_hardreset(ap, class);
+       rc = sata_std_hardreset(link, class, deadline);
 
-       ahci_start_engine(port_mmio);
+       ahci_start_engine(ap);
 
-       if (rc == 0 && ata_port_online(ap))
+       if (rc == 0 && ata_link_online(link))
                *class = ahci_dev_classify(ap);
-       if (*class == ATA_DEV_UNKNOWN)
+       if (rc != -EAGAIN && *class == ATA_DEV_UNKNOWN)
                *class = ATA_DEV_NONE;
 
        DPRINTK("EXIT, rc=%d, class=%u\n", rc, *class);
        return rc;
 }
 
-static void ahci_postreset(struct ata_port *ap, unsigned int *class)
+static int ahci_vt8251_hardreset(struct ata_link *link, unsigned int *class,
+                                unsigned long deadline)
+{
+       struct ata_port *ap = link->ap;
+       u32 serror;
+       int rc;
+
+       DPRINTK("ENTER\n");
+
+       ahci_stop_engine(ap);
+
+       rc = sata_link_hardreset(link, sata_ehc_deb_timing(&link->eh_context),
+                                deadline);
+
+       /* vt8251 needs SError cleared for the port to operate */
+       ahci_scr_read(ap, SCR_ERROR, &serror);
+       ahci_scr_write(ap, SCR_ERROR, serror);
+
+       ahci_start_engine(ap);
+
+       DPRINTK("EXIT, rc=%d, class=%u\n", rc, *class);
+
+       /* vt8251 doesn't clear BSY on signature FIS reception,
+        * request follow-up softreset.
+        */
+       return rc ?: -EAGAIN;
+}
+
+static void ahci_postreset(struct ata_link *link, unsigned int *class)
 {
-       void __iomem *port_mmio = (void __iomem *) ap->ioaddr.cmd_addr;
+       struct ata_port *ap = link->ap;
+       void __iomem *port_mmio = ahci_port_base(ap);
        u32 new_tmp, tmp;
 
-       ata_std_postreset(ap, class);
+       ata_std_postreset(link, class);
 
        /* Make sure port's ATAPI bit is set appropriately */
        new_tmp = tmp = readl(port_mmio + PORT_CMD);
@@ -883,9 +1233,15 @@ static void ahci_postreset(struct ata_port *ap, unsigned int *class)
        }
 }
 
+static int ahci_pmp_softreset(struct ata_link *link, unsigned int *class,
+                             unsigned long deadline)
+{
+       return ahci_do_softreset(link, class, link->pmp, deadline);
+}
+
 static u8 ahci_check_status(struct ata_port *ap)
 {
-       void __iomem *mmio = (void __iomem *) ap->ioaddr.cmd_addr;
+       void __iomem *mmio = ap->ioaddr.cmd_addr;
 
        return readl(mmio + PORT_TFDATA) & 0xFF;
 }
@@ -941,7 +1297,7 @@ static void ahci_qc_prep(struct ata_queued_cmd *qc)
         */
        cmd_tbl = pp->cmd_tbl + qc->tag * AHCI_CMD_TBL_SZ;
 
-       ata_tf_to_fis(&qc->tf, cmd_tbl, 0);
+       ata_tf_to_fis(&qc->tf, qc->dev->link->pmp, 1, cmd_tbl);
        if (is_atapi) {
                memset(cmd_tbl + AHCI_CMD_TBL_CDB, 0, 32);
                memcpy(cmd_tbl + AHCI_CMD_TBL_CDB, qc->cdb, qc->dev->cdb_len);
@@ -954,7 +1310,7 @@ static void ahci_qc_prep(struct ata_queued_cmd *qc)
        /*
         * Fill in command slot information.
         */
-       opts = cmd_fis_len | n_elem << 16;
+       opts = cmd_fis_len | n_elem << 16 | (qc->dev->link->pmp << 12);
        if (qc->tf.flags & ATA_TFLAG_WRITE)
                opts |= AHCI_CMD_WRITE;
        if (is_atapi)
@@ -965,63 +1321,87 @@ static void ahci_qc_prep(struct ata_queued_cmd *qc)
 
 static void ahci_error_intr(struct ata_port *ap, u32 irq_stat)
 {
+       struct ahci_host_priv *hpriv = ap->host->private_data;
        struct ahci_port_priv *pp = ap->private_data;
-       struct ata_eh_info *ehi = &ap->eh_info;
-       unsigned int err_mask = 0, action = 0;
-       struct ata_queued_cmd *qc;
+       struct ata_eh_info *host_ehi = &ap->link.eh_info;
+       struct ata_link *link = NULL;
+       struct ata_queued_cmd *active_qc;
+       struct ata_eh_info *active_ehi;
        u32 serror;
 
-       ata_ehi_clear_desc(ehi);
+       /* determine active link */
+       ata_port_for_each_link(link, ap)
+               if (ata_link_active(link))
+                       break;
+       if (!link)
+               link = &ap->link;
+
+       active_qc = ata_qc_from_tag(ap, link->active_tag);
+       active_ehi = &link->eh_info;
+
+       /* record irq stat */
+       ata_ehi_clear_desc(host_ehi);
+       ata_ehi_push_desc(host_ehi, "irq_stat 0x%08x", irq_stat);
 
        /* AHCI needs SError cleared; otherwise, it might lock up */
-       serror = ahci_scr_read(ap, SCR_ERROR);
+       ahci_scr_read(ap, SCR_ERROR, &serror);
        ahci_scr_write(ap, SCR_ERROR, serror);
-
-       /* analyze @irq_stat */
-       ata_ehi_push_desc(ehi, "irq_stat 0x%08x", irq_stat);
+       host_ehi->serror |= serror;
 
        /* some controllers set IRQ_IF_ERR on device errors, ignore it */
-       if (ap->flags & AHCI_FLAG_IGN_IRQ_IF_ERR)
+       if (hpriv->flags & AHCI_HFLAG_IGN_IRQ_IF_ERR)
                irq_stat &= ~PORT_IRQ_IF_ERR;
 
-       if (irq_stat & PORT_IRQ_TF_ERR)
-               err_mask |= AC_ERR_DEV;
+       if (irq_stat & PORT_IRQ_TF_ERR) {
+               /* If qc is active, charge it; otherwise, the active
+                * link.  There's no active qc on NCQ errors.  It will
+                * be determined by EH by reading log page 10h.
+                */
+               if (active_qc)
+                       active_qc->err_mask |= AC_ERR_DEV;
+               else
+                       active_ehi->err_mask |= AC_ERR_DEV;
+
+               if (hpriv->flags & AHCI_HFLAG_IGN_SERR_INTERNAL)
+                       host_ehi->serror &= ~SERR_INTERNAL;
+       }
+
+       if (irq_stat & PORT_IRQ_UNK_FIS) {
+               u32 *unk = (u32 *)(pp->rx_fis + RX_FIS_UNK);
+
+               active_ehi->err_mask |= AC_ERR_HSM;
+               active_ehi->action |= ATA_EH_SOFTRESET;
+               ata_ehi_push_desc(active_ehi,
+                                 "unknown FIS %08x %08x %08x %08x" ,
+                                 unk[0], unk[1], unk[2], unk[3]);
+       }
+
+       if (ap->nr_pmp_links && (irq_stat & PORT_IRQ_BAD_PMP)) {
+               active_ehi->err_mask |= AC_ERR_HSM;
+               active_ehi->action |= ATA_EH_SOFTRESET;
+               ata_ehi_push_desc(active_ehi, "incorrect PMP");
+       }
 
        if (irq_stat & (PORT_IRQ_HBUS_ERR | PORT_IRQ_HBUS_DATA_ERR)) {
-               err_mask |= AC_ERR_HOST_BUS;
-               action |= ATA_EH_SOFTRESET;
+               host_ehi->err_mask |= AC_ERR_HOST_BUS;
+               host_ehi->action |= ATA_EH_SOFTRESET;
+               ata_ehi_push_desc(host_ehi, "host bus error");
        }
 
        if (irq_stat & PORT_IRQ_IF_ERR) {
-               err_mask |= AC_ERR_ATA_BUS;
-               action |= ATA_EH_SOFTRESET;
-               ata_ehi_push_desc(ehi, ", interface fatal error");
+               host_ehi->err_mask |= AC_ERR_ATA_BUS;
+               host_ehi->action |= ATA_EH_SOFTRESET;
+               ata_ehi_push_desc(host_ehi, "interface fatal error");
        }
 
        if (irq_stat & (PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY)) {
-               ata_ehi_hotplugged(ehi);
-               ata_ehi_push_desc(ehi, ", %s", irq_stat & PORT_IRQ_CONNECT ?
+               ata_ehi_hotplugged(host_ehi);
+               ata_ehi_push_desc(host_ehi, "%s",
+                       irq_stat & PORT_IRQ_CONNECT ?
                        "connection status changed" : "PHY RDY changed");
        }
 
-       if (irq_stat & PORT_IRQ_UNK_FIS) {
-               u32 *unk = (u32 *)(pp->rx_fis + RX_FIS_UNK);
-
-               err_mask |= AC_ERR_HSM;
-               action |= ATA_EH_SOFTRESET;
-               ata_ehi_push_desc(ehi, ", unknown FIS %08x %08x %08x %08x",
-                                 unk[0], unk[1], unk[2], unk[3]);
-       }
-
        /* okay, let's hand over to EH */
-       ehi->serror |= serror;
-       ehi->action |= action;
-
-       qc = ata_qc_from_tag(ap, ap->active_tag);
-       if (qc)
-               qc->err_mask |= err_mask;
-       else
-               ehi->err_mask |= err_mask;
 
        if (irq_stat & PORT_IRQ_FREEZE)
                ata_port_freeze(ap);
@@ -1029,28 +1409,67 @@ static void ahci_error_intr(struct ata_port *ap, u32 irq_stat)
                ata_port_abort(ap);
 }
 
-static void ahci_host_intr(struct ata_port *ap)
+static void ahci_port_intr(struct ata_port *ap)
 {
-       void __iomem *mmio = ap->host->mmio_base;
-       void __iomem *port_mmio = ahci_port_base(mmio, ap->port_no);
-       struct ata_eh_info *ehi = &ap->eh_info;
+       void __iomem *port_mmio = ap->ioaddr.cmd_addr;
+       struct ata_eh_info *ehi = &ap->link.eh_info;
+       struct ahci_port_priv *pp = ap->private_data;
+       struct ahci_host_priv *hpriv = ap->host->private_data;
+       int resetting = !!(ap->pflags & ATA_PFLAG_RESETTING);
        u32 status, qc_active;
-       int rc;
+       int rc, known_irq = 0;
 
        status = readl(port_mmio + PORT_IRQ_STAT);
        writel(status, port_mmio + PORT_IRQ_STAT);
 
+       /* ignore BAD_PMP while resetting */
+       if (unlikely(resetting))
+               status &= ~PORT_IRQ_BAD_PMP;
+
        if (unlikely(status & PORT_IRQ_ERROR)) {
                ahci_error_intr(ap, status);
                return;
        }
 
-       if (ap->sactive)
+       if (status & PORT_IRQ_SDB_FIS) {
+               /* If SNotification is available, leave notification
+                * handling to sata_async_notification().  If not,
+                * emulate it by snooping SDB FIS RX area.
+                *
+                * Snooping FIS RX area is probably cheaper than
+                * poking SNotification but some constrollers which
+                * implement SNotification, ICH9 for example, don't
+                * store AN SDB FIS into receive area.
+                */
+               if (hpriv->cap & HOST_CAP_SNTF)
+                       sata_async_notification(ap);
+               else {
+                       /* If the 'N' bit in word 0 of the FIS is set,
+                        * we just received asynchronous notification.
+                        * Tell libata about it.
+                        */
+                       const __le32 *f = pp->rx_fis + RX_FIS_SDB;
+                       u32 f0 = le32_to_cpu(f[0]);
+
+                       if (f0 & (1 << 15))
+                               sata_async_notification(ap);
+               }
+       }
+
+       /* pp->active_link is valid iff any command is in flight */
+       if (ap->qc_active && pp->active_link->sactive)
                qc_active = readl(port_mmio + PORT_SCR_ACT);
        else
                qc_active = readl(port_mmio + PORT_CMD_ISSUE);
 
        rc = ata_qc_complete_multiple(ap, qc_active, NULL);
+
+       /* If resetting, spurious or invalid completions are expected,
+        * return unconditionally.
+        */
+       if (resetting)
+               return;
+
        if (rc > 0)
                return;
        if (rc < 0) {
@@ -1060,20 +1479,65 @@ static void ahci_host_intr(struct ata_port *ap)
                return;
        }
 
-       /* hmmm... a spurious interupt */
+       /* hmmm... a spurious interrupt */
 
-       /* some devices send D2H reg with I bit set during NCQ command phase */
-       if (ap->sactive && (status & PORT_IRQ_D2H_REG_FIS))
+       /* if !NCQ, ignore.  No modern ATA device has broken HSM
+        * implementation for non-NCQ commands.
+        */
+       if (!ap->link.sactive)
                return;
 
-       /* ignore interim PIO setup fis interrupts */
-       if (ata_tag_valid(ap->active_tag) && (status & PORT_IRQ_PIOS_FIS))
-               return;
+       if (status & PORT_IRQ_D2H_REG_FIS) {
+               if (!pp->ncq_saw_d2h)
+                       ata_port_printk(ap, KERN_INFO,
+                               "D2H reg with I during NCQ, "
+                               "this message won't be printed again\n");
+               pp->ncq_saw_d2h = 1;
+               known_irq = 1;
+       }
+
+       if (status & PORT_IRQ_DMAS_FIS) {
+               if (!pp->ncq_saw_dmas)
+                       ata_port_printk(ap, KERN_INFO,
+                               "DMAS FIS during NCQ, "
+                               "this message won't be printed again\n");
+               pp->ncq_saw_dmas = 1;
+               known_irq = 1;
+       }
+
+       if (status & PORT_IRQ_SDB_FIS) {
+               const __le32 *f = pp->rx_fis + RX_FIS_SDB;
+
+               if (le32_to_cpu(f[1])) {
+                       /* SDB FIS containing spurious completions
+                        * might be dangerous, whine and fail commands
+                        * with HSM violation.  EH will turn off NCQ
+                        * after several such failures.
+                        */
+                       ata_ehi_push_desc(ehi,
+                               "spurious completions during NCQ "
+                               "issue=0x%x SAct=0x%x FIS=%08x:%08x",
+                               readl(port_mmio + PORT_CMD_ISSUE),
+                               readl(port_mmio + PORT_SCR_ACT),
+                               le32_to_cpu(f[0]), le32_to_cpu(f[1]));
+                       ehi->err_mask |= AC_ERR_HSM;
+                       ehi->action |= ATA_EH_SOFTRESET;
+                       ata_port_freeze(ap);
+               } else {
+                       if (!pp->ncq_saw_sdb)
+                               ata_port_printk(ap, KERN_INFO,
+                                       "spurious SDB FIS %08x:%08x during NCQ, "
+                                       "this message won't be printed again\n",
+                                       le32_to_cpu(f[0]), le32_to_cpu(f[1]));
+                       pp->ncq_saw_sdb = 1;
+               }
+               known_irq = 1;
+       }
 
-       if (ata_ratelimit())
+       if (!known_irq)
                ata_port_printk(ap, KERN_INFO, "spurious interrupt "
-                               "(irq_stat 0x%x active_tag %d sactive 0x%x)\n",
-                               status, ap->active_tag, ap->sactive);
+                               "(irq_stat 0x%x active_tag 0x%x sactive 0x%x)\n",
+                               status, ap->link.active_tag, ap->link.sactive);
 }
 
 static void ahci_irq_clear(struct ata_port *ap)
@@ -1092,7 +1556,7 @@ static irqreturn_t ahci_interrupt(int irq, void *dev_instance)
        VPRINTK("ENTER\n");
 
        hpriv = host->private_data;
-       mmio = host->mmio_base;
+       mmio = host->iomap[AHCI_PCI_BAR];
 
        /* sigh.  0xffffffff is a valid return from h/w */
        irq_stat = readl(mmio + HOST_IRQ_STAT);
@@ -1110,7 +1574,7 @@ static irqreturn_t ahci_interrupt(int irq, void *dev_instance)
 
                ap = host->ports[i];
                if (ap) {
-                       ahci_host_intr(ap);
+                       ahci_port_intr(ap);
                        VPRINTK("port %u\n", i);
                } else {
                        VPRINTK("port %u (no irq)\n", i);
@@ -1137,7 +1601,14 @@ static irqreturn_t ahci_interrupt(int irq, void *dev_instance)
 static unsigned int ahci_qc_issue(struct ata_queued_cmd *qc)
 {
        struct ata_port *ap = qc->ap;
-       void __iomem *port_mmio = (void __iomem *) ap->ioaddr.cmd_addr;
+       void __iomem *port_mmio = ahci_port_base(ap);
+       struct ahci_port_priv *pp = ap->private_data;
+
+       /* Keep track of the currently active link.  It will be used
+        * in completion path to determine whether NCQ phase is in
+        * progress.
+        */
+       pp->active_link = qc->dev->link;
 
        if (qc->tf.protocol == ATA_PROT_NCQ)
                writel(1 << qc->tag, port_mmio + PORT_SCR_ACT);
@@ -1149,8 +1620,7 @@ static unsigned int ahci_qc_issue(struct ata_queued_cmd *qc)
 
 static void ahci_freeze(struct ata_port *ap)
 {
-       void __iomem *mmio = ap->host->mmio_base;
-       void __iomem *port_mmio = ahci_port_base(mmio, ap->port_no);
+       void __iomem *port_mmio = ahci_port_base(ap);
 
        /* turn IRQ off */
        writel(0, port_mmio + PORT_IRQ_MASK);
@@ -1158,86 +1628,119 @@ static void ahci_freeze(struct ata_port *ap)
 
 static void ahci_thaw(struct ata_port *ap)
 {
-       void __iomem *mmio = ap->host->mmio_base;
-       void __iomem *port_mmio = ahci_port_base(mmio, ap->port_no);
+       void __iomem *mmio = ap->host->iomap[AHCI_PCI_BAR];
+       void __iomem *port_mmio = ahci_port_base(ap);
        u32 tmp;
+       struct ahci_port_priv *pp = ap->private_data;
 
        /* clear IRQ */
        tmp = readl(port_mmio + PORT_IRQ_STAT);
        writel(tmp, port_mmio + PORT_IRQ_STAT);
-       writel(1 << ap->id, mmio + HOST_IRQ_STAT);
+       writel(1 << ap->port_no, mmio + HOST_IRQ_STAT);
 
        /* turn IRQ back on */
-       writel(DEF_PORT_IRQ, port_mmio + PORT_IRQ_MASK);
+       writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
 }
 
 static void ahci_error_handler(struct ata_port *ap)
 {
-       void __iomem *mmio = ap->host->mmio_base;
-       void __iomem *port_mmio = ahci_port_base(mmio, ap->port_no);
+       if (!(ap->pflags & ATA_PFLAG_FROZEN)) {
+               /* restart engine */
+               ahci_stop_engine(ap);
+               ahci_start_engine(ap);
+       }
+
+       /* perform recovery */
+       sata_pmp_do_eh(ap, ata_std_prereset, ahci_softreset,
+                      ahci_hardreset, ahci_postreset,
+                      sata_pmp_std_prereset, ahci_pmp_softreset,
+                      sata_pmp_std_hardreset, sata_pmp_std_postreset);
+}
 
+static void ahci_vt8251_error_handler(struct ata_port *ap)
+{
        if (!(ap->pflags & ATA_PFLAG_FROZEN)) {
                /* restart engine */
-               ahci_stop_engine(port_mmio);
-               ahci_start_engine(port_mmio);
+               ahci_stop_engine(ap);
+               ahci_start_engine(ap);
        }
 
        /* perform recovery */
-       ata_do_eh(ap, ata_std_prereset, ahci_softreset, ahci_hardreset,
+       ata_do_eh(ap, ata_std_prereset, ahci_softreset, ahci_vt8251_hardreset,
                  ahci_postreset);
 }
 
 static void ahci_post_internal_cmd(struct ata_queued_cmd *qc)
 {
        struct ata_port *ap = qc->ap;
-       void __iomem *mmio = ap->host->mmio_base;
-       void __iomem *port_mmio = ahci_port_base(mmio, ap->port_no);
 
+       /* make DMA engine forget about the failed command */
        if (qc->flags & ATA_QCFLAG_FAILED)
-               qc->err_mask |= AC_ERR_OTHER;
+               ahci_kick_engine(ap, 1);
+}
 
-       if (qc->err_mask) {
-               /* make DMA engine forget about the failed command */
-               ahci_stop_engine(port_mmio);
-               ahci_start_engine(port_mmio);
-       }
+static void ahci_pmp_attach(struct ata_port *ap)
+{
+       void __iomem *port_mmio = ahci_port_base(ap);
+       struct ahci_port_priv *pp = ap->private_data;
+       u32 cmd;
+
+       cmd = readl(port_mmio + PORT_CMD);
+       cmd |= PORT_CMD_PMP;
+       writel(cmd, port_mmio + PORT_CMD);
+
+       pp->intr_mask |= PORT_IRQ_BAD_PMP;
+       writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
 }
 
-static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg)
+static void ahci_pmp_detach(struct ata_port *ap)
 {
-       struct ahci_host_priv *hpriv = ap->host->private_data;
+       void __iomem *port_mmio = ahci_port_base(ap);
        struct ahci_port_priv *pp = ap->private_data;
-       void __iomem *mmio = ap->host->mmio_base;
-       void __iomem *port_mmio = ahci_port_base(mmio, ap->port_no);
-       const char *emsg = NULL;
-       int rc;
+       u32 cmd;
 
-       rc = ahci_deinit_port(port_mmio, hpriv->cap, &emsg);
-       if (rc) {
-               ata_port_printk(ap, KERN_ERR, "%s (%d)\n", emsg, rc);
-               ahci_init_port(port_mmio, hpriv->cap,
-                              pp->cmd_slot_dma, pp->rx_fis_dma);
-       }
+       cmd = readl(port_mmio + PORT_CMD);
+       cmd &= ~PORT_CMD_PMP;
+       writel(cmd, port_mmio + PORT_CMD);
 
-       return rc;
+       pp->intr_mask &= ~PORT_IRQ_BAD_PMP;
+       writel(pp->intr_mask, port_mmio + PORT_IRQ_MASK);
 }
 
 static int ahci_port_resume(struct ata_port *ap)
 {
-       struct ahci_port_priv *pp = ap->private_data;
-       struct ahci_host_priv *hpriv = ap->host->private_data;
-       void __iomem *mmio = ap->host->mmio_base;
-       void __iomem *port_mmio = ahci_port_base(mmio, ap->port_no);
+       ahci_power_up(ap);
+       ahci_start_port(ap);
 
-       ahci_init_port(port_mmio, hpriv->cap, pp->cmd_slot_dma, pp->rx_fis_dma);
+       if (ap->nr_pmp_links)
+               ahci_pmp_attach(ap);
+       else
+               ahci_pmp_detach(ap);
 
        return 0;
 }
 
+#ifdef CONFIG_PM
+static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg)
+{
+       const char *emsg = NULL;
+       int rc;
+
+       rc = ahci_deinit_port(ap, &emsg);
+       if (rc == 0)
+               ahci_power_down(ap);
+       else {
+               ata_port_printk(ap, KERN_ERR, "%s (%d)\n", emsg, rc);
+               ahci_start_port(ap);
+       }
+
+       return rc;
+}
+
 static int ahci_pci_device_suspend(struct pci_dev *pdev, pm_message_t mesg)
 {
        struct ata_host *host = dev_get_drvdata(&pdev->dev);
-       void __iomem *mmio = host->mmio_base;
+       void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
        u32 ctl;
 
        if (mesg.event == PM_EVENT_SUSPEND) {
@@ -1257,53 +1760,46 @@ static int ahci_pci_device_suspend(struct pci_dev *pdev, pm_message_t mesg)
 static int ahci_pci_device_resume(struct pci_dev *pdev)
 {
        struct ata_host *host = dev_get_drvdata(&pdev->dev);
-       struct ahci_host_priv *hpriv = host->private_data;
-       void __iomem *mmio = host->mmio_base;
        int rc;
 
-       ata_pci_device_do_resume(pdev);
+       rc = ata_pci_device_do_resume(pdev);
+       if (rc)
+               return rc;
 
        if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) {
-               rc = ahci_reset_controller(mmio, pdev);
+               rc = ahci_reset_controller(host);
                if (rc)
                        return rc;
 
-               ahci_init_controller(mmio, pdev, host->n_ports, hpriv->cap);
+               ahci_init_controller(host);
        }
 
        ata_host_resume(host);
 
        return 0;
 }
+#endif
 
 static int ahci_port_start(struct ata_port *ap)
 {
        struct device *dev = ap->host->dev;
-       struct ahci_host_priv *hpriv = ap->host->private_data;
        struct ahci_port_priv *pp;
-       void __iomem *mmio = ap->host->mmio_base;
-       void __iomem *port_mmio = ahci_port_base(mmio, ap->port_no);
        void *mem;
        dma_addr_t mem_dma;
        int rc;
 
-       pp = kmalloc(sizeof(*pp), GFP_KERNEL);
+       pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
        if (!pp)
                return -ENOMEM;
-       memset(pp, 0, sizeof(*pp));
 
        rc = ata_pad_alloc(ap, dev);
-       if (rc) {
-               kfree(pp);
+       if (rc)
                return rc;
-       }
 
-       mem = dma_alloc_coherent(dev, AHCI_PORT_PRIV_DMA_SZ, &mem_dma, GFP_KERNEL);
-       if (!mem) {
-               ata_pad_free(ap, dev);
-               kfree(pp);
+       mem = dmam_alloc_coherent(dev, AHCI_PORT_PRIV_DMA_SZ, &mem_dma,
+                                 GFP_KERNEL);
+       if (!mem)
                return -ENOMEM;
-       }
        memset(mem, 0, AHCI_PORT_PRIV_DMA_SZ);
 
        /*
@@ -1332,69 +1828,33 @@ static int ahci_port_start(struct ata_port *ap)
        pp->cmd_tbl = mem;
        pp->cmd_tbl_dma = mem_dma;
 
-       ap->private_data = pp;
+       /*
+        * Save off initial list of interrupts to be enabled.
+        * This could be changed later
+        */
+       pp->intr_mask = DEF_PORT_IRQ;
 
-       /* initialize port */
-       ahci_init_port(port_mmio, hpriv->cap, pp->cmd_slot_dma, pp->rx_fis_dma);
+       ap->private_data = pp;
 
-       return 0;
+       /* engage engines, captain */
+       return ahci_port_resume(ap);
 }
 
 static void ahci_port_stop(struct ata_port *ap)
 {
-       struct device *dev = ap->host->dev;
-       struct ahci_host_priv *hpriv = ap->host->private_data;
-       struct ahci_port_priv *pp = ap->private_data;
-       void __iomem *mmio = ap->host->mmio_base;
-       void __iomem *port_mmio = ahci_port_base(mmio, ap->port_no);
        const char *emsg = NULL;
        int rc;
 
        /* de-initialize port */
-       rc = ahci_deinit_port(port_mmio, hpriv->cap, &emsg);
+       rc = ahci_deinit_port(ap, &emsg);
        if (rc)
                ata_port_printk(ap, KERN_WARNING, "%s (%d)\n", emsg, rc);
-
-       ap->private_data = NULL;
-       dma_free_coherent(dev, AHCI_PORT_PRIV_DMA_SZ,
-                         pp->cmd_slot, pp->cmd_slot_dma);
-       ata_pad_free(ap, dev);
-       kfree(pp);
-}
-
-static void ahci_setup_port(struct ata_ioports *port, unsigned long base,
-                           unsigned int port_idx)
-{
-       VPRINTK("ENTER, base==0x%lx, port_idx %u\n", base, port_idx);
-       base = ahci_port_base_ul(base, port_idx);
-       VPRINTK("base now==0x%lx\n", base);
-
-       port->cmd_addr          = base;
-       port->scr_addr          = base + PORT_SCR;
-
-       VPRINTK("EXIT\n");
 }
 
-static int ahci_host_init(struct ata_probe_ent *probe_ent)
+static int ahci_configure_dma_masks(struct pci_dev *pdev, int using_dac)
 {
-       struct ahci_host_priv *hpriv = probe_ent->private_data;
-       struct pci_dev *pdev = to_pci_dev(probe_ent->dev);
-       void __iomem *mmio = probe_ent->mmio_base;
-       unsigned int i, using_dac;
        int rc;
 
-       rc = ahci_reset_controller(mmio, pdev);
-       if (rc)
-               return rc;
-
-       hpriv->cap = readl(mmio + HOST_CAP);
-       hpriv->port_map = readl(mmio + HOST_PORTS_IMPL);
-       probe_ent->n_ports = (hpriv->cap & 0x1f) + 1;
-
-       VPRINTK("cap 0x%x  port_map 0x%x  n_ports %d\n",
-               hpriv->cap, hpriv->port_map, probe_ent->n_ports);
-
-       using_dac = hpriv->cap & HOST_CAP_64;
        if (using_dac &&
            !pci_set_dma_mask(pdev, DMA_64BIT_MASK)) {
                rc = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
@@ -1420,22 +1880,14 @@ static int ahci_host_init(struct ata_probe_ent *probe_ent)
                        return rc;
                }
        }
-
-       for (i = 0; i < probe_ent->n_ports; i++)
-               ahci_setup_port(&probe_ent->port[i], (unsigned long) mmio, i);
-
-       ahci_init_controller(mmio, pdev, probe_ent->n_ports, hpriv->cap);
-
-       pci_set_master(pdev);
-
        return 0;
 }
 
-static void ahci_print_info(struct ata_probe_ent *probe_ent)
+static void ahci_print_info(struct ata_host *host)
 {
-       struct ahci_host_priv *hpriv = probe_ent->private_data;
-       struct pci_dev *pdev = to_pci_dev(probe_ent->dev);
-       void __iomem *mmio = probe_ent->mmio_base;
+       struct ahci_host_priv *hpriv = host->private_data;
+       struct pci_dev *pdev = to_pci_dev(host->dev);
+       void __iomem *mmio = host->iomap[AHCI_PCI_BAR];
        u32 vers, cap, impl, speed;
        const char *speed_s;
        u16 cc;
@@ -1454,11 +1906,11 @@ static void ahci_print_info(struct ata_probe_ent *probe_ent)
                speed_s = "?";
 
        pci_read_config_word(pdev, 0x0a, &cc);
-       if (cc == 0x0101)
+       if (cc == PCI_CLASS_STORAGE_IDE)
                scc_s = "IDE";
-       else if (cc == 0x0106)
+       else if (cc == PCI_CLASS_STORAGE_SATA)
                scc_s = "SATA";
-       else if (cc == 0x0104)
+       else if (cc == PCI_CLASS_STORAGE_RAID)
                scc_s = "RAID";
        else
                scc_s = "unknown";
@@ -1481,12 +1933,13 @@ static void ahci_print_info(struct ata_probe_ent *probe_ent)
 
        dev_printk(KERN_INFO, &pdev->dev,
                "flags: "
-               "%s%s%s%s%s%s"
-               "%s%s%s%s%s%s%s\n"
+               "%s%s%s%s%s%s%s"
+               "%s%s%s%s%s%s%s\n"
                ,
 
                cap & (1 << 31) ? "64bit " : "",
                cap & (1 << 30) ? "ncq " : "",
+               cap & (1 << 29) ? "sntf " : "",
                cap & (1 << 28) ? "ilck " : "",
                cap & (1 << 27) ? "stag " : "",
                cap & (1 << 26) ? "pm " : "",
@@ -1502,16 +1955,15 @@ static void ahci_print_info(struct ata_probe_ent *probe_ent)
                );
 }
 
-static int ahci_init_one (struct pci_dev *pdev, const struct pci_device_id *ent)
+static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
        static int printed_version;
-       struct ata_probe_ent *probe_ent = NULL;
+       struct ata_port_info pi = ahci_port_info[ent->driver_data];
+       const struct ata_port_info *ppi[] = { &pi, NULL };
+       struct device *dev = &pdev->dev;
        struct ahci_host_priv *hpriv;
-       unsigned long base;
-       void __iomem *mmio_base;
-       unsigned int board_idx = (unsigned int) ent->driver_data;
-       int have_msi, pci_dev_busy = 0;
-       int rc;
+       struct ata_host *host;
+       int i, rc;
 
        VPRINTK("ENTER\n");
 
@@ -1520,139 +1972,73 @@ static int ahci_init_one (struct pci_dev *pdev, const struct pci_device_id *ent)
        if (!printed_version++)
                dev_printk(KERN_DEBUG, &pdev->dev, "version " DRV_VERSION "\n");
 
-       /* JMicron-specific fixup: make sure we're in AHCI mode */
-       /* This is protected from races with ata_jmicron by the pci probe
-          locking */
-       if (pdev->vendor == PCI_VENDOR_ID_JMICRON) {
-               /* AHCI enable, AHCI on function 0 */
-               pci_write_config_byte(pdev, 0x41, 0xa1);
-               /* Function 1 is the PATA controller */
-               if (PCI_FUNC(pdev->devfn))
-                       return -ENODEV;
-       }
+       /* acquire resources */
+       rc = pcim_enable_device(pdev);
+       if (rc)
+               return rc;
 
-       rc = pci_enable_device(pdev);
+       rc = pcim_iomap_regions(pdev, 1 << AHCI_PCI_BAR, DRV_NAME);
+       if (rc == -EBUSY)
+               pcim_pin_device(pdev);
        if (rc)
                return rc;
 
-       rc = pci_request_regions(pdev, DRV_NAME);
-       if (rc) {
-               pci_dev_busy = 1;
-               goto err_out;
-       }
+       hpriv = devm_kzalloc(dev, sizeof(*hpriv), GFP_KERNEL);
+       if (!hpriv)
+               return -ENOMEM;
+       hpriv->flags |= (unsigned long)pi.private_data;
 
-       if (pci_enable_msi(pdev) == 0)
-               have_msi = 1;
-       else {
+       if ((hpriv->flags & AHCI_HFLAG_NO_MSI) || pci_enable_msi(pdev))
                pci_intx(pdev, 1);
-               have_msi = 0;
-       }
 
-       probe_ent = kmalloc(sizeof(*probe_ent), GFP_KERNEL);
-       if (probe_ent == NULL) {
-               rc = -ENOMEM;
-               goto err_out_msi;
-       }
+       /* save initial config */
+       ahci_save_initial_config(pdev, hpriv);
 
-       memset(probe_ent, 0, sizeof(*probe_ent));
-       probe_ent->dev = pci_dev_to_dev(pdev);
-       INIT_LIST_HEAD(&probe_ent->node);
+       /* prepare host */
+       if (hpriv->cap & HOST_CAP_NCQ)
+               pi.flags |= ATA_FLAG_NCQ;
 
-       mmio_base = pci_iomap(pdev, AHCI_PCI_BAR, 0);
-       if (mmio_base == NULL) {
-               rc = -ENOMEM;
-               goto err_out_free_ent;
-       }
-       base = (unsigned long) mmio_base;
+       if (hpriv->cap & HOST_CAP_PMP)
+               pi.flags |= ATA_FLAG_PMP;
 
-       hpriv = kmalloc(sizeof(*hpriv), GFP_KERNEL);
-       if (!hpriv) {
-               rc = -ENOMEM;
-               goto err_out_iounmap;
-       }
-       memset(hpriv, 0, sizeof(*hpriv));
+       host = ata_host_alloc_pinfo(&pdev->dev, ppi, fls(hpriv->port_map));
+       if (!host)
+               return -ENOMEM;
+       host->iomap = pcim_iomap_table(pdev);
+       host->private_data = hpriv;
+
+       for (i = 0; i < host->n_ports; i++) {
+               struct ata_port *ap = host->ports[i];
+               void __iomem *port_mmio = ahci_port_base(ap);
 
-       probe_ent->sht          = ahci_port_info[board_idx].sht;
-       probe_ent->port_flags   = ahci_port_info[board_idx].flags;
-       probe_ent->pio_mask     = ahci_port_info[board_idx].pio_mask;
-       probe_ent->udma_mask    = ahci_port_info[board_idx].udma_mask;
-       probe_ent->port_ops     = ahci_port_info[board_idx].port_ops;
+               ata_port_pbar_desc(ap, AHCI_PCI_BAR, -1, "abar");
+               ata_port_pbar_desc(ap, AHCI_PCI_BAR,
+                                  0x100 + ap->port_no * 0x80, "port");
 
-               probe_ent->irq = pdev->irq;
-               probe_ent->irq_flags = IRQF_SHARED;
-       probe_ent->mmio_base = mmio_base;
-       probe_ent->private_data = hpriv;
+               /* standard SATA port setup */
+               if (hpriv->port_map & (1 << i))
+                       ap->ioaddr.cmd_addr = port_mmio;
 
-       if (have_msi)
-               hpriv->flags |= AHCI_FLAG_MSI;
+               /* disabled/not-implemented port */
+               else
+                       ap->ops = &ata_dummy_port_ops;
+       }
 
        /* initialize adapter */
-       rc = ahci_host_init(probe_ent);
+       rc = ahci_configure_dma_masks(pdev, hpriv->cap & HOST_CAP_64);
        if (rc)
-               goto err_out_hpriv;
-
-       if (!(probe_ent->port_flags & AHCI_FLAG_NO_NCQ) &&
-           (hpriv->cap & HOST_CAP_NCQ))
-               probe_ent->port_flags |= ATA_FLAG_NCQ;
-
-       ahci_print_info(probe_ent);
-
-       /* FIXME: check ata_device_add return value */
-       ata_device_add(probe_ent);
-       kfree(probe_ent);
-
-       return 0;
-
-err_out_hpriv:
-       kfree(hpriv);
-err_out_iounmap:
-       pci_iounmap(pdev, mmio_base);
-err_out_free_ent:
-       kfree(probe_ent);
-err_out_msi:
-       if (have_msi)
-               pci_disable_msi(pdev);
-       else
-               pci_intx(pdev, 0);
-       pci_release_regions(pdev);
-err_out:
-       if (!pci_dev_busy)
-               pci_disable_device(pdev);
-       return rc;
-}
-
-static void ahci_remove_one (struct pci_dev *pdev)
-{
-       struct device *dev = pci_dev_to_dev(pdev);
-       struct ata_host *host = dev_get_drvdata(dev);
-       struct ahci_host_priv *hpriv = host->private_data;
-       unsigned int i;
-       int have_msi;
-
-       for (i = 0; i < host->n_ports; i++)
-               ata_port_detach(host->ports[i]);
-
-       have_msi = hpriv->flags & AHCI_FLAG_MSI;
-       free_irq(host->irq, host);
-
-       for (i = 0; i < host->n_ports; i++) {
-               struct ata_port *ap = host->ports[i];
+               return rc;
 
-               ata_scsi_release(ap->scsi_host);
-               scsi_host_put(ap->scsi_host);
-       }
+       rc = ahci_reset_controller(host);
+       if (rc)
+               return rc;
 
-       kfree(hpriv);
-       pci_iounmap(pdev, host->mmio_base);
-       kfree(host);
+       ahci_init_controller(host);
+       ahci_print_info(host);
 
-       if (have_msi)
-               pci_disable_msi(pdev);
-       else
-               pci_intx(pdev, 0);
-       pci_release_regions(pdev);
-       pci_disable_device(pdev);
-       dev_set_drvdata(dev, NULL);
+       pci_set_master(pdev);
+       return ata_host_activate(host, pdev->irq, ahci_interrupt, IRQF_SHARED,
+                                &ahci_sht);
 }
 
 static int __init ahci_init(void)