Merge branch 'next' of git://git.kernel.org/pub/scm/linux/kernel/git/djbw/async_tx...
NeilBrown [Wed, 23 Sep 2009 08:31:11 +0000 (18:31 +1000)]
61 files changed:
Documentation/crypto/async-tx-api.txt
arch/arm/include/asm/hardware/iop3xx-adma.h
arch/arm/include/asm/hardware/iop_adma.h
arch/arm/mach-iop13xx/include/mach/adma.h
arch/arm/mach-iop13xx/setup.c
arch/arm/plat-iop/adma.c
arch/powerpc/include/asm/fsldma.h [new file with mode: 0644]
arch/sh/drivers/dma/Kconfig
arch/sh/drivers/dma/Makefile
arch/sh/include/asm/dma-sh.h
crypto/async_tx/Kconfig
crypto/async_tx/Makefile
crypto/async_tx/async_memcpy.c
crypto/async_tx/async_memset.c
crypto/async_tx/async_pq.c [new file with mode: 0644]
crypto/async_tx/async_raid6_recov.c [new file with mode: 0644]
crypto/async_tx/async_tx.c
crypto/async_tx/async_xor.c
crypto/async_tx/raid6test.c [new file with mode: 0644]
drivers/dca/dca-core.c
drivers/dma/Kconfig
drivers/dma/Makefile
drivers/dma/at_hdmac.c
drivers/dma/at_hdmac_regs.h
drivers/dma/dmaengine.c
drivers/dma/dmatest.c
drivers/dma/dw_dmac.c
drivers/dma/dw_dmac_regs.h
drivers/dma/fsldma.c
drivers/dma/fsldma.h
drivers/dma/ioat.c [deleted file]
drivers/dma/ioat/Makefile [new file with mode: 0644]
drivers/dma/ioat/dca.c [moved from drivers/dma/ioat_dca.c with 98% similarity]
drivers/dma/ioat/dma.c [new file with mode: 0644]
drivers/dma/ioat/dma.h [new file with mode: 0644]
drivers/dma/ioat/dma_v2.c [new file with mode: 0644]
drivers/dma/ioat/dma_v2.h [new file with mode: 0644]
drivers/dma/ioat/dma_v3.c [new file with mode: 0644]
drivers/dma/ioat/hw.h [new file with mode: 0644]
drivers/dma/ioat/pci.c [new file with mode: 0644]
drivers/dma/ioat/registers.h [moved from drivers/dma/ioatdma_registers.h with 84% similarity]
drivers/dma/ioat_dma.c [deleted file]
drivers/dma/ioatdma.h [deleted file]
drivers/dma/ioatdma_hw.h [deleted file]
drivers/dma/iop-adma.c
drivers/dma/iovlock.c
drivers/dma/mv_xor.c
drivers/dma/mv_xor.h
drivers/dma/shdma.c [new file with mode: 0644]
drivers/dma/shdma.h [new file with mode: 0644]
drivers/dma/txx9dmac.c
drivers/dma/txx9dmac.h
drivers/idle/i7300_idle.c
drivers/md/Kconfig
drivers/md/raid5.c
drivers/md/raid5.h
drivers/mmc/host/atmel-mci.c
include/linux/async_tx.h
include/linux/dca.h
include/linux/dmaengine.h
include/linux/pci_ids.h

index 9f59fcb..ba046b8 100644 (file)
@@ -54,20 +54,23 @@ features surfaced as a result:
 
 3.1 General format of the API:
 struct dma_async_tx_descriptor *
-async_<operation>(<op specific parameters>,
-                 enum async_tx_flags flags,
-                 struct dma_async_tx_descriptor *dependency,
-                 dma_async_tx_callback callback_routine,
-                 void *callback_parameter);
+async_<operation>(<op specific parameters>, struct async_submit ctl *submit)
 
 3.2 Supported operations:
-memcpy       - memory copy between a source and a destination buffer
-memset       - fill a destination buffer with a byte value
-xor          - xor a series of source buffers and write the result to a
-              destination buffer
-xor_zero_sum - xor a series of source buffers and set a flag if the
-              result is zero.  The implementation attempts to prevent
-              writes to memory
+memcpy  - memory copy between a source and a destination buffer
+memset  - fill a destination buffer with a byte value
+xor     - xor a series of source buffers and write the result to a
+         destination buffer
+xor_val - xor a series of source buffers and set a flag if the
+         result is zero.  The implementation attempts to prevent
+         writes to memory
+pq     - generate the p+q (raid6 syndrome) from a series of source buffers
+pq_val  - validate that a p and or q buffer are in sync with a given series of
+         sources
+datap  - (raid6_datap_recov) recover a raid6 data block and the p block
+         from the given sources
+2data  - (raid6_2data_recov) recover 2 raid6 data blocks from the given
+         sources
 
 3.3 Descriptor management:
 The return value is non-NULL and points to a 'descriptor' when the operation
@@ -80,8 +83,8 @@ acknowledged by the application before the offload engine driver is allowed to
 recycle (or free) the descriptor.  A descriptor can be acked by one of the
 following methods:
 1/ setting the ASYNC_TX_ACK flag if no child operations are to be submitted
-2/ setting the ASYNC_TX_DEP_ACK flag to acknowledge the parent
-   descriptor of a new operation.
+2/ submitting an unacknowledged descriptor as a dependency to another
+   async_tx call will implicitly set the acknowledged state.
 3/ calling async_tx_ack() on the descriptor.
 
 3.4 When does the operation execute?
@@ -119,30 +122,42 @@ of an operation.
 Perform a xor->copy->xor operation where each operation depends on the
 result from the previous operation:
 
-void complete_xor_copy_xor(void *param)
+void callback(void *param)
 {
-       printk("complete\n");
+       struct completion *cmp = param;
+
+       complete(cmp);
 }
 
-int run_xor_copy_xor(struct page **xor_srcs,
-                    int xor_src_cnt,
-                    struct page *xor_dest,
-                    size_t xor_len,
-                    struct page *copy_src,
-                    struct page *copy_dest,
-                    size_t copy_len)
+void run_xor_copy_xor(struct page **xor_srcs,
+                     int xor_src_cnt,
+                     struct page *xor_dest,
+                     size_t xor_len,
+                     struct page *copy_src,
+                     struct page *copy_dest,
+                     size_t copy_len)
 {
        struct dma_async_tx_descriptor *tx;
+       addr_conv_t addr_conv[xor_src_cnt];
+       struct async_submit_ctl submit;
+       addr_conv_t addr_conv[NDISKS];
+       struct completion cmp;
+
+       init_async_submit(&submit, ASYNC_TX_XOR_DROP_DST, NULL, NULL, NULL,
+                         addr_conv);
+       tx = async_xor(xor_dest, xor_srcs, 0, xor_src_cnt, xor_len, &submit)
 
-       tx = async_xor(xor_dest, xor_srcs, 0, xor_src_cnt, xor_len,
-                      ASYNC_TX_XOR_DROP_DST, NULL, NULL, NULL);
-       tx = async_memcpy(copy_dest, copy_src, 0, 0, copy_len,
-                         ASYNC_TX_DEP_ACK, tx, NULL, NULL);
-       tx = async_xor(xor_dest, xor_srcs, 0, xor_src_cnt, xor_len,
-                      ASYNC_TX_XOR_DROP_DST | ASYNC_TX_DEP_ACK | ASYNC_TX_ACK,
-                      tx, complete_xor_copy_xor, NULL);
+       submit->depend_tx = tx;
+       tx = async_memcpy(copy_dest, copy_src, 0, 0, copy_len, &submit);
+
+       init_completion(&cmp);
+       init_async_submit(&submit, ASYNC_TX_XOR_DROP_DST | ASYNC_TX_ACK, tx,
+                         callback, &cmp, addr_conv);
+       tx = async_xor(xor_dest, xor_srcs, 0, xor_src_cnt, xor_len, &submit);
 
        async_tx_issue_pending_all();
+
+       wait_for_completion(&cmp);
 }
 
 See include/linux/async_tx.h for more information on the flags.  See the
index 83e6ba3..1a8c727 100644 (file)
@@ -187,11 +187,74 @@ union iop3xx_desc {
        void *ptr;
 };
 
+/* No support for p+q operations */
+static inline int
+iop_chan_pq_slot_count(size_t len, int src_cnt, int *slots_per_op)
+{
+       BUG();
+       return 0;
+}
+
+static inline void
+iop_desc_init_pq(struct iop_adma_desc_slot *desc, int src_cnt,
+                 unsigned long flags)
+{
+       BUG();
+}
+
+static inline void
+iop_desc_set_pq_addr(struct iop_adma_desc_slot *desc, dma_addr_t *addr)
+{
+       BUG();
+}
+
+static inline void
+iop_desc_set_pq_src_addr(struct iop_adma_desc_slot *desc, int src_idx,
+                        dma_addr_t addr, unsigned char coef)
+{
+       BUG();
+}
+
+static inline int
+iop_chan_pq_zero_sum_slot_count(size_t len, int src_cnt, int *slots_per_op)
+{
+       BUG();
+       return 0;
+}
+
+static inline void
+iop_desc_init_pq_zero_sum(struct iop_adma_desc_slot *desc, int src_cnt,
+                         unsigned long flags)
+{
+       BUG();
+}
+
+static inline void
+iop_desc_set_pq_zero_sum_byte_count(struct iop_adma_desc_slot *desc, u32 len)
+{
+       BUG();
+}
+
+#define iop_desc_set_pq_zero_sum_src_addr iop_desc_set_pq_src_addr
+
+static inline void
+iop_desc_set_pq_zero_sum_addr(struct iop_adma_desc_slot *desc, int pq_idx,
+                             dma_addr_t *src)
+{
+       BUG();
+}
+
 static inline int iop_adma_get_max_xor(void)
 {
        return 32;
 }
 
+static inline int iop_adma_get_max_pq(void)
+{
+       BUG();
+       return 0;
+}
+
 static inline u32 iop_chan_get_current_descriptor(struct iop_adma_chan *chan)
 {
        int id = chan->device->id;
@@ -332,6 +395,11 @@ static inline int iop_chan_zero_sum_slot_count(size_t len, int src_cnt,
        return slot_cnt;
 }
 
+static inline int iop_desc_is_pq(struct iop_adma_desc_slot *desc)
+{
+       return 0;
+}
+
 static inline u32 iop_desc_get_dest_addr(struct iop_adma_desc_slot *desc,
                                        struct iop_adma_chan *chan)
 {
@@ -349,6 +417,14 @@ static inline u32 iop_desc_get_dest_addr(struct iop_adma_desc_slot *desc,
        return 0;
 }
 
+
+static inline u32 iop_desc_get_qdest_addr(struct iop_adma_desc_slot *desc,
+                                         struct iop_adma_chan *chan)
+{
+       BUG();
+       return 0;
+}
+
 static inline u32 iop_desc_get_byte_count(struct iop_adma_desc_slot *desc,
                                        struct iop_adma_chan *chan)
 {
@@ -756,13 +832,14 @@ static inline void iop_desc_set_block_fill_val(struct iop_adma_desc_slot *desc,
        hw_desc->src[0] = val;
 }
 
-static inline int iop_desc_get_zero_result(struct iop_adma_desc_slot *desc)
+static inline enum sum_check_flags
+iop_desc_get_zero_result(struct iop_adma_desc_slot *desc)
 {
        struct iop3xx_desc_aau *hw_desc = desc->hw_desc;
        struct iop3xx_aau_desc_ctrl desc_ctrl = hw_desc->desc_ctrl_field;
 
        iop_paranoia(!(desc_ctrl.tx_complete && desc_ctrl.zero_result_en));
-       return desc_ctrl.zero_result_err;
+       return desc_ctrl.zero_result_err << SUM_CHECK_P;
 }
 
 static inline void iop_chan_append(struct iop_adma_chan *chan)
index 385c6e8..59b8c38 100644 (file)
@@ -86,6 +86,7 @@ struct iop_adma_chan {
  * @idx: pool index
  * @unmap_src_cnt: number of xor sources
  * @unmap_len: transaction bytecount
+ * @tx_list: list of descriptors that are associated with one operation
  * @async_tx: support for the async_tx api
  * @group_list: list of slots that make up a multi-descriptor transaction
  *     for example transfer lengths larger than the supported hw max
@@ -102,10 +103,12 @@ struct iop_adma_desc_slot {
        u16 idx;
        u16 unmap_src_cnt;
        size_t unmap_len;
+       struct list_head tx_list;
        struct dma_async_tx_descriptor async_tx;
        union {
                u32 *xor_check_result;
                u32 *crc32_result;
+               u32 *pq_check_result;
        };
 };
 
index 5722e86..6d3782d 100644 (file)
@@ -150,6 +150,8 @@ static inline int iop_adma_get_max_xor(void)
        return 16;
 }
 
+#define iop_adma_get_max_pq iop_adma_get_max_xor
+
 static inline u32 iop_chan_get_current_descriptor(struct iop_adma_chan *chan)
 {
        return __raw_readl(ADMA_ADAR(chan));
@@ -211,7 +213,10 @@ iop_chan_xor_slot_count(size_t len, int src_cnt, int *slots_per_op)
 #define IOP_ADMA_MAX_BYTE_COUNT ADMA_MAX_BYTE_COUNT
 #define IOP_ADMA_ZERO_SUM_MAX_BYTE_COUNT ADMA_MAX_BYTE_COUNT
 #define IOP_ADMA_XOR_MAX_BYTE_COUNT ADMA_MAX_BYTE_COUNT
+#define IOP_ADMA_PQ_MAX_BYTE_COUNT ADMA_MAX_BYTE_COUNT
 #define iop_chan_zero_sum_slot_count(l, s, o) iop_chan_xor_slot_count(l, s, o)
+#define iop_chan_pq_slot_count iop_chan_xor_slot_count
+#define iop_chan_pq_zero_sum_slot_count iop_chan_xor_slot_count
 
 static inline u32 iop_desc_get_dest_addr(struct iop_adma_desc_slot *desc,
                                        struct iop_adma_chan *chan)
@@ -220,6 +225,13 @@ static inline u32 iop_desc_get_dest_addr(struct iop_adma_desc_slot *desc,
        return hw_desc->dest_addr;
 }
 
+static inline u32 iop_desc_get_qdest_addr(struct iop_adma_desc_slot *desc,
+                                         struct iop_adma_chan *chan)
+{
+       struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc;
+       return hw_desc->q_dest_addr;
+}
+
 static inline u32 iop_desc_get_byte_count(struct iop_adma_desc_slot *desc,
                                        struct iop_adma_chan *chan)
 {
@@ -319,6 +331,58 @@ iop_desc_init_zero_sum(struct iop_adma_desc_slot *desc, int src_cnt,
        return 1;
 }
 
+static inline void
+iop_desc_init_pq(struct iop_adma_desc_slot *desc, int src_cnt,
+                 unsigned long flags)
+{
+       struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc;
+       union {
+               u32 value;
+               struct iop13xx_adma_desc_ctrl field;
+       } u_desc_ctrl;
+
+       u_desc_ctrl.value = 0;
+       u_desc_ctrl.field.src_select = src_cnt - 1;
+       u_desc_ctrl.field.xfer_dir = 3; /* local to internal bus */
+       u_desc_ctrl.field.pq_xfer_en = 1;
+       u_desc_ctrl.field.p_xfer_dis = !!(flags & DMA_PREP_PQ_DISABLE_P);
+       u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT;
+       hw_desc->desc_ctrl = u_desc_ctrl.value;
+}
+
+static inline int iop_desc_is_pq(struct iop_adma_desc_slot *desc)
+{
+       struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc;
+       union {
+               u32 value;
+               struct iop13xx_adma_desc_ctrl field;
+       } u_desc_ctrl;
+
+       u_desc_ctrl.value = hw_desc->desc_ctrl;
+       return u_desc_ctrl.field.pq_xfer_en;
+}
+
+static inline void
+iop_desc_init_pq_zero_sum(struct iop_adma_desc_slot *desc, int src_cnt,
+                         unsigned long flags)
+{
+       struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc;
+       union {
+               u32 value;
+               struct iop13xx_adma_desc_ctrl field;
+       } u_desc_ctrl;
+
+       u_desc_ctrl.value = 0;
+       u_desc_ctrl.field.src_select = src_cnt - 1;
+       u_desc_ctrl.field.xfer_dir = 3; /* local to internal bus */
+       u_desc_ctrl.field.zero_result = 1;
+       u_desc_ctrl.field.status_write_back_en = 1;
+       u_desc_ctrl.field.pq_xfer_en = 1;
+       u_desc_ctrl.field.p_xfer_dis = !!(flags & DMA_PREP_PQ_DISABLE_P);
+       u_desc_ctrl.field.int_en = flags & DMA_PREP_INTERRUPT;
+       hw_desc->desc_ctrl = u_desc_ctrl.value;
+}
+
 static inline void iop_desc_set_byte_count(struct iop_adma_desc_slot *desc,
                                        struct iop_adma_chan *chan,
                                        u32 byte_count)
@@ -351,6 +415,7 @@ iop_desc_set_zero_sum_byte_count(struct iop_adma_desc_slot *desc, u32 len)
        }
 }
 
+#define iop_desc_set_pq_zero_sum_byte_count iop_desc_set_zero_sum_byte_count
 
 static inline void iop_desc_set_dest_addr(struct iop_adma_desc_slot *desc,
                                        struct iop_adma_chan *chan,
@@ -361,6 +426,16 @@ static inline void iop_desc_set_dest_addr(struct iop_adma_desc_slot *desc,
        hw_desc->upper_dest_addr = 0;
 }
 
+static inline void
+iop_desc_set_pq_addr(struct iop_adma_desc_slot *desc, dma_addr_t *addr)
+{
+       struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc;
+
+       hw_desc->dest_addr = addr[0];
+       hw_desc->q_dest_addr = addr[1];
+       hw_desc->upper_dest_addr = 0;
+}
+
 static inline void iop_desc_set_memcpy_src_addr(struct iop_adma_desc_slot *desc,
                                        dma_addr_t addr)
 {
@@ -389,6 +464,29 @@ static inline void iop_desc_set_xor_src_addr(struct iop_adma_desc_slot *desc,
 }
 
 static inline void
+iop_desc_set_pq_src_addr(struct iop_adma_desc_slot *desc, int src_idx,
+                        dma_addr_t addr, unsigned char coef)
+{
+       int slot_cnt = desc->slot_cnt, slots_per_op = desc->slots_per_op;
+       struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc, *iter;
+       struct iop13xx_adma_src *src;
+       int i = 0;
+
+       do {
+               iter = iop_hw_desc_slot_idx(hw_desc, i);
+               src = &iter->src[src_idx];
+               src->src_addr = addr;
+               src->pq_upper_src_addr = 0;
+               src->pq_dmlt = coef;
+               slot_cnt -= slots_per_op;
+               if (slot_cnt) {
+                       i += slots_per_op;
+                       addr += IOP_ADMA_PQ_MAX_BYTE_COUNT;
+               }
+       } while (slot_cnt);
+}
+
+static inline void
 iop_desc_init_interrupt(struct iop_adma_desc_slot *desc,
        struct iop_adma_chan *chan)
 {
@@ -399,6 +497,15 @@ iop_desc_init_interrupt(struct iop_adma_desc_slot *desc,
 }
 
 #define iop_desc_set_zero_sum_src_addr iop_desc_set_xor_src_addr
+#define iop_desc_set_pq_zero_sum_src_addr iop_desc_set_pq_src_addr
+
+static inline void
+iop_desc_set_pq_zero_sum_addr(struct iop_adma_desc_slot *desc, int pq_idx,
+                             dma_addr_t *src)
+{
+       iop_desc_set_xor_src_addr(desc, pq_idx, src[pq_idx]);
+       iop_desc_set_xor_src_addr(desc, pq_idx+1, src[pq_idx+1]);
+}
 
 static inline void iop_desc_set_next_desc(struct iop_adma_desc_slot *desc,
                                        u32 next_desc_addr)
@@ -428,18 +535,20 @@ static inline void iop_desc_set_block_fill_val(struct iop_adma_desc_slot *desc,
        hw_desc->block_fill_data = val;
 }
 
-static inline int iop_desc_get_zero_result(struct iop_adma_desc_slot *desc)
+static inline enum sum_check_flags
+iop_desc_get_zero_result(struct iop_adma_desc_slot *desc)
 {
        struct iop13xx_adma_desc_hw *hw_desc = desc->hw_desc;
        struct iop13xx_adma_desc_ctrl desc_ctrl = hw_desc->desc_ctrl_field;
        struct iop13xx_adma_byte_count byte_count = hw_desc->byte_count_field;
+       enum sum_check_flags flags;
 
        BUG_ON(!(byte_count.tx_complete && desc_ctrl.zero_result));
 
-       if (desc_ctrl.pq_xfer_en)
-               return byte_count.zero_result_err_q;
-       else
-               return byte_count.zero_result_err;
+       flags = byte_count.zero_result_err_q << SUM_CHECK_Q;
+       flags |= byte_count.zero_result_err << SUM_CHECK_P;
+
+       return flags;
 }
 
 static inline void iop_chan_append(struct iop_adma_chan *chan)
index bee42c6..5c147fb 100644 (file)
@@ -477,10 +477,8 @@ void __init iop13xx_platform_init(void)
                        plat_data = &iop13xx_adma_0_data;
                        dma_cap_set(DMA_MEMCPY, plat_data->cap_mask);
                        dma_cap_set(DMA_XOR, plat_data->cap_mask);
-                       dma_cap_set(DMA_DUAL_XOR, plat_data->cap_mask);
-                       dma_cap_set(DMA_ZERO_SUM, plat_data->cap_mask);
+                       dma_cap_set(DMA_XOR_VAL, plat_data->cap_mask);
                        dma_cap_set(DMA_MEMSET, plat_data->cap_mask);
-                       dma_cap_set(DMA_MEMCPY_CRC32C, plat_data->cap_mask);
                        dma_cap_set(DMA_INTERRUPT, plat_data->cap_mask);
                        break;
                case IOP13XX_INIT_ADMA_1:
@@ -489,10 +487,8 @@ void __init iop13xx_platform_init(void)
                        plat_data = &iop13xx_adma_1_data;
                        dma_cap_set(DMA_MEMCPY, plat_data->cap_mask);
                        dma_cap_set(DMA_XOR, plat_data->cap_mask);
-                       dma_cap_set(DMA_DUAL_XOR, plat_data->cap_mask);
-                       dma_cap_set(DMA_ZERO_SUM, plat_data->cap_mask);
+                       dma_cap_set(DMA_XOR_VAL, plat_data->cap_mask);
                        dma_cap_set(DMA_MEMSET, plat_data->cap_mask);
-                       dma_cap_set(DMA_MEMCPY_CRC32C, plat_data->cap_mask);
                        dma_cap_set(DMA_INTERRUPT, plat_data->cap_mask);
                        break;
                case IOP13XX_INIT_ADMA_2:
@@ -501,14 +497,11 @@ void __init iop13xx_platform_init(void)
                        plat_data = &iop13xx_adma_2_data;
                        dma_cap_set(DMA_MEMCPY, plat_data->cap_mask);
                        dma_cap_set(DMA_XOR, plat_data->cap_mask);
-                       dma_cap_set(DMA_DUAL_XOR, plat_data->cap_mask);
-                       dma_cap_set(DMA_ZERO_SUM, plat_data->cap_mask);
+                       dma_cap_set(DMA_XOR_VAL, plat_data->cap_mask);
                        dma_cap_set(DMA_MEMSET, plat_data->cap_mask);
-                       dma_cap_set(DMA_MEMCPY_CRC32C, plat_data->cap_mask);
                        dma_cap_set(DMA_INTERRUPT, plat_data->cap_mask);
-                       dma_cap_set(DMA_PQ_XOR, plat_data->cap_mask);
-                       dma_cap_set(DMA_PQ_UPDATE, plat_data->cap_mask);
-                       dma_cap_set(DMA_PQ_ZERO_SUM, plat_data->cap_mask);
+                       dma_cap_set(DMA_PQ, plat_data->cap_mask);
+                       dma_cap_set(DMA_PQ_VAL, plat_data->cap_mask);
                        break;
                }
        }
index 3c127aa..1ff6a37 100644 (file)
@@ -179,7 +179,6 @@ static int __init iop3xx_adma_cap_init(void)
        dma_cap_set(DMA_INTERRUPT, iop3xx_dma_0_data.cap_mask);
        #else
        dma_cap_set(DMA_MEMCPY, iop3xx_dma_0_data.cap_mask);
-       dma_cap_set(DMA_MEMCPY_CRC32C, iop3xx_dma_0_data.cap_mask);
        dma_cap_set(DMA_INTERRUPT, iop3xx_dma_0_data.cap_mask);
        #endif
 
@@ -188,7 +187,6 @@ static int __init iop3xx_adma_cap_init(void)
        dma_cap_set(DMA_INTERRUPT, iop3xx_dma_1_data.cap_mask);
        #else
        dma_cap_set(DMA_MEMCPY, iop3xx_dma_1_data.cap_mask);
-       dma_cap_set(DMA_MEMCPY_CRC32C, iop3xx_dma_1_data.cap_mask);
        dma_cap_set(DMA_INTERRUPT, iop3xx_dma_1_data.cap_mask);
        #endif
 
@@ -198,7 +196,7 @@ static int __init iop3xx_adma_cap_init(void)
        dma_cap_set(DMA_INTERRUPT, iop3xx_aau_data.cap_mask);
        #else
        dma_cap_set(DMA_XOR, iop3xx_aau_data.cap_mask);
-       dma_cap_set(DMA_ZERO_SUM, iop3xx_aau_data.cap_mask);
+       dma_cap_set(DMA_XOR_VAL, iop3xx_aau_data.cap_mask);
        dma_cap_set(DMA_MEMSET, iop3xx_aau_data.cap_mask);
        dma_cap_set(DMA_INTERRUPT, iop3xx_aau_data.cap_mask);
        #endif
diff --git a/arch/powerpc/include/asm/fsldma.h b/arch/powerpc/include/asm/fsldma.h
new file mode 100644 (file)
index 0000000..a67aeed
--- /dev/null
@@ -0,0 +1,136 @@
+/*
+ * Freescale MPC83XX / MPC85XX DMA Controller
+ *
+ * Copyright (c) 2009 Ira W. Snyder <iws@ovro.caltech.edu>
+ *
+ * This file is licensed under the terms of the GNU General Public License
+ * version 2. This program is licensed "as is" without any warranty of any
+ * kind, whether express or implied.
+ */
+
+#ifndef __ARCH_POWERPC_ASM_FSLDMA_H__
+#define __ARCH_POWERPC_ASM_FSLDMA_H__
+
+#include <linux/dmaengine.h>
+
+/*
+ * Definitions for the Freescale DMA controller's DMA_SLAVE implemention
+ *
+ * The Freescale DMA_SLAVE implementation was designed to handle many-to-many
+ * transfers. An example usage would be an accelerated copy between two
+ * scatterlists. Another example use would be an accelerated copy from
+ * multiple non-contiguous device buffers into a single scatterlist.
+ *
+ * A DMA_SLAVE transaction is defined by a struct fsl_dma_slave. This
+ * structure contains a list of hardware addresses that should be copied
+ * to/from the scatterlist passed into device_prep_slave_sg(). The structure
+ * also has some fields to enable hardware-specific features.
+ */
+
+/**
+ * struct fsl_dma_hw_addr
+ * @entry: linked list entry
+ * @address: the hardware address
+ * @length: length to transfer
+ *
+ * Holds a single physical hardware address / length pair for use
+ * with the DMAEngine DMA_SLAVE API.
+ */
+struct fsl_dma_hw_addr {
+       struct list_head entry;
+
+       dma_addr_t address;
+       size_t length;
+};
+
+/**
+ * struct fsl_dma_slave
+ * @addresses: a linked list of struct fsl_dma_hw_addr structures
+ * @request_count: value for DMA request count
+ * @src_loop_size: setup and enable constant source-address DMA transfers
+ * @dst_loop_size: setup and enable constant destination address DMA transfers
+ * @external_start: enable externally started DMA transfers
+ * @external_pause: enable externally paused DMA transfers
+ *
+ * Holds a list of address / length pairs for use with the DMAEngine
+ * DMA_SLAVE API implementation for the Freescale DMA controller.
+ */
+struct fsl_dma_slave {
+
+       /* List of hardware address/length pairs */
+       struct list_head addresses;
+
+       /* Support for extra controller features */
+       unsigned int request_count;
+       unsigned int src_loop_size;
+       unsigned int dst_loop_size;
+       bool external_start;
+       bool external_pause;
+};
+
+/**
+ * fsl_dma_slave_append - add an address/length pair to a struct fsl_dma_slave
+ * @slave: the &struct fsl_dma_slave to add to
+ * @address: the hardware address to add
+ * @length: the length of bytes to transfer from @address
+ *
+ * Add a hardware address/length pair to a struct fsl_dma_slave. Returns 0 on
+ * success, -ERRNO otherwise.
+ */
+static inline int fsl_dma_slave_append(struct fsl_dma_slave *slave,
+                                      dma_addr_t address, size_t length)
+{
+       struct fsl_dma_hw_addr *addr;
+
+       addr = kzalloc(sizeof(*addr), GFP_ATOMIC);
+       if (!addr)
+               return -ENOMEM;
+
+       INIT_LIST_HEAD(&addr->entry);
+       addr->address = address;
+       addr->length = length;
+
+       list_add_tail(&addr->entry, &slave->addresses);
+       return 0;
+}
+
+/**
+ * fsl_dma_slave_free - free a struct fsl_dma_slave
+ * @slave: the struct fsl_dma_slave to free
+ *
+ * Free a struct fsl_dma_slave and all associated address/length pairs
+ */
+static inline void fsl_dma_slave_free(struct fsl_dma_slave *slave)
+{
+       struct fsl_dma_hw_addr *addr, *tmp;
+
+       if (slave) {
+               list_for_each_entry_safe(addr, tmp, &slave->addresses, entry) {
+                       list_del(&addr->entry);
+                       kfree(addr);
+               }
+
+               kfree(slave);
+       }
+}
+
+/**
+ * fsl_dma_slave_alloc - allocate a struct fsl_dma_slave
+ * @gfp: the flags to pass to kmalloc when allocating this structure
+ *
+ * Allocate a struct fsl_dma_slave for use by the DMA_SLAVE API. Returns a new
+ * struct fsl_dma_slave on success, or NULL on failure.
+ */
+static inline struct fsl_dma_slave *fsl_dma_slave_alloc(gfp_t gfp)
+{
+       struct fsl_dma_slave *slave;
+
+       slave = kzalloc(sizeof(*slave), gfp);
+       if (!slave)
+               return NULL;
+
+       INIT_LIST_HEAD(&slave->addresses);
+       return slave;
+}
+
+#endif /* __ARCH_POWERPC_ASM_FSLDMA_H__ */
index b91fa8d..4d58eb0 100644 (file)
@@ -1,12 +1,9 @@
 menu "DMA support"
 
-config SH_DMA_API
-       bool
 
 config SH_DMA
        bool "SuperH on-chip DMA controller (DMAC) support"
        depends on CPU_SH3 || CPU_SH4
-       select SH_DMA_API
        default n
 
 config SH_DMA_IRQ_MULTI
@@ -19,6 +16,15 @@ config SH_DMA_IRQ_MULTI
                     CPU_SUBTYPE_SH7780  || CPU_SUBTYPE_SH7785  || \
                     CPU_SUBTYPE_SH7760
 
+config SH_DMA_API
+       depends on SH_DMA
+       bool "SuperH DMA API support"
+       default n
+       help
+         SH_DMA_API always enabled DMA API of used SuperH.
+         If you want to use DMA ENGINE, you must not enable this.
+         Please enable DMA_ENGINE and SH_DMAE.
+
 config NR_ONCHIP_DMA_CHANNELS
        int
        depends on SH_DMA
index c606813..d88c948 100644 (file)
@@ -2,8 +2,7 @@
 # Makefile for the SuperH DMA specific kernel interface routines under Linux.
 #
 
-obj-$(CONFIG_SH_DMA_API)       += dma-api.o dma-sysfs.o
-obj-$(CONFIG_SH_DMA)           += dma-sh.o
+obj-$(CONFIG_SH_DMA_API)       += dma-sh.o dma-api.o dma-sysfs.o
 obj-$(CONFIG_PVR2_DMA)         += dma-pvr2.o
 obj-$(CONFIG_G2_DMA)           += dma-g2.o
 obj-$(CONFIG_SH_DMABRG)                += dmabrg.o
index 68a5f4c..78eed3e 100644 (file)
@@ -116,4 +116,17 @@ static u32 dma_base_addr[] __maybe_unused = {
 #define CHCR    0x0C
 #define DMAOR  0x40
 
+/*
+ * for dma engine
+ *
+ * SuperH DMA mode
+ */
+#define SHDMA_MIX_IRQ  (1 << 1)
+#define SHDMA_DMAOR1   (1 << 2)
+#define SHDMA_DMAE1            (1 << 3)
+
+struct sh_dmae_pdata {
+       unsigned int mode;
+};
+
 #endif /* __DMA_SH_H */
index d8fb391..e5aeb2b 100644 (file)
@@ -14,3 +14,12 @@ config ASYNC_MEMSET
        tristate
        select ASYNC_CORE
 
+config ASYNC_PQ
+       tristate
+       select ASYNC_CORE
+
+config ASYNC_RAID6_RECOV
+       tristate
+       select ASYNC_CORE
+       select ASYNC_PQ
+
index 27baa7d..d1e0e6f 100644 (file)
@@ -2,3 +2,6 @@ obj-$(CONFIG_ASYNC_CORE) += async_tx.o
 obj-$(CONFIG_ASYNC_MEMCPY) += async_memcpy.o
 obj-$(CONFIG_ASYNC_MEMSET) += async_memset.o
 obj-$(CONFIG_ASYNC_XOR) += async_xor.o
+obj-$(CONFIG_ASYNC_PQ) += async_pq.o
+obj-$(CONFIG_ASYNC_RAID6_RECOV) += async_raid6_recov.o
+obj-$(CONFIG_ASYNC_RAID6_TEST) += raid6test.o
index ddccfb0..0ec1fb6 100644 (file)
  * async_memcpy - attempt to copy memory with a dma engine.
  * @dest: destination page
  * @src: src page
- * @offset: offset in pages to start transaction
+ * @dest_offset: offset into 'dest' to start transaction
+ * @src_offset: offset into 'src' to start transaction
  * @len: length in bytes
- * @flags: ASYNC_TX_ACK, ASYNC_TX_DEP_ACK,
- * @depend_tx: memcpy depends on the result of this transaction
- * @cb_fn: function to call when the memcpy completes
- * @cb_param: parameter to pass to the callback routine
+ * @submit: submission / completion modifiers
+ *
+ * honored flags: ASYNC_TX_ACK
  */
 struct dma_async_tx_descriptor *
 async_memcpy(struct page *dest, struct page *src, unsigned int dest_offset,
-       unsigned int src_offset, size_t len, enum async_tx_flags flags,
-       struct dma_async_tx_descriptor *depend_tx,
-       dma_async_tx_callback cb_fn, void *cb_param)
+            unsigned int src_offset, size_t len,
+            struct async_submit_ctl *submit)
 {
-       struct dma_chan *chan = async_tx_find_channel(depend_tx, DMA_MEMCPY,
+       struct dma_chan *chan = async_tx_find_channel(submit, DMA_MEMCPY,
                                                      &dest, 1, &src, 1, len);
        struct dma_device *device = chan ? chan->device : NULL;
        struct dma_async_tx_descriptor *tx = NULL;
 
-       if (device) {
+       if (device && is_dma_copy_aligned(device, src_offset, dest_offset, len)) {
                dma_addr_t dma_dest, dma_src;
-               unsigned long dma_prep_flags = cb_fn ? DMA_PREP_INTERRUPT : 0;
+               unsigned long dma_prep_flags = 0;
 
+               if (submit->cb_fn)
+                       dma_prep_flags |= DMA_PREP_INTERRUPT;
+               if (submit->flags & ASYNC_TX_FENCE)
+                       dma_prep_flags |= DMA_PREP_FENCE;
                dma_dest = dma_map_page(device->dev, dest, dest_offset, len,
                                        DMA_FROM_DEVICE);
 
@@ -67,13 +70,13 @@ async_memcpy(struct page *dest, struct page *src, unsigned int dest_offset,
 
        if (tx) {
                pr_debug("%s: (async) len: %zu\n", __func__, len);
-               async_tx_submit(chan, tx, flags, depend_tx, cb_fn, cb_param);
+               async_tx_submit(chan, tx, submit);
        } else {
                void *dest_buf, *src_buf;
                pr_debug("%s: (sync) len: %zu\n", __func__, len);
 
                /* wait for any prerequisite operations */
-               async_tx_quiesce(&depend_tx);
+               async_tx_quiesce(&submit->depend_tx);
 
                dest_buf = kmap_atomic(dest, KM_USER0) + dest_offset;
                src_buf = kmap_atomic(src, KM_USER1) + src_offset;
@@ -83,26 +86,13 @@ async_memcpy(struct page *dest, struct page *src, unsigned int dest_offset,
                kunmap_atomic(dest_buf, KM_USER0);
                kunmap_atomic(src_buf, KM_USER1);
 
-               async_tx_sync_epilog(cb_fn, cb_param);
+               async_tx_sync_epilog(submit);
        }
 
        return tx;
 }
 EXPORT_SYMBOL_GPL(async_memcpy);
 
-static int __init async_memcpy_init(void)
-{
-       return 0;
-}
-
-static void __exit async_memcpy_exit(void)
-{
-       do { } while (0);
-}
-
-module_init(async_memcpy_init);
-module_exit(async_memcpy_exit);
-
 MODULE_AUTHOR("Intel Corporation");
 MODULE_DESCRIPTION("asynchronous memcpy api");
 MODULE_LICENSE("GPL");
index 5b5eb99..58e4a87 100644 (file)
  * @val: fill value
  * @offset: offset in pages to start transaction
  * @len: length in bytes
- * @flags: ASYNC_TX_ACK, ASYNC_TX_DEP_ACK
- * @depend_tx: memset depends on the result of this transaction
- * @cb_fn: function to call when the memcpy completes
- * @cb_param: parameter to pass to the callback routine
+ *
+ * honored flags: ASYNC_TX_ACK
  */
 struct dma_async_tx_descriptor *
-async_memset(struct page *dest, int val, unsigned int offset,
-       size_t len, enum async_tx_flags flags,
-       struct dma_async_tx_descriptor *depend_tx,
-       dma_async_tx_callback cb_fn, void *cb_param)
+async_memset(struct page *dest, int val, unsigned int offset, size_t len,
+            struct async_submit_ctl *submit)
 {
-       struct dma_chan *chan = async_tx_find_channel(depend_tx, DMA_MEMSET,
+       struct dma_chan *chan = async_tx_find_channel(submit, DMA_MEMSET,
                                                      &dest, 1, NULL, 0, len);
        struct dma_device *device = chan ? chan->device : NULL;
        struct dma_async_tx_descriptor *tx = NULL;
 
-       if (device) {
+       if (device && is_dma_fill_aligned(device, offset, 0, len)) {
                dma_addr_t dma_dest;
-               unsigned long dma_prep_flags = cb_fn ? DMA_PREP_INTERRUPT : 0;
+               unsigned long dma_prep_flags = 0;
 
+               if (submit->cb_fn)
+                       dma_prep_flags |= DMA_PREP_INTERRUPT;
+               if (submit->flags & ASYNC_TX_FENCE)
+                       dma_prep_flags |= DMA_PREP_FENCE;
                dma_dest = dma_map_page(device->dev, dest, offset, len,
                                        DMA_FROM_DEVICE);
 
@@ -64,38 +64,25 @@ async_memset(struct page *dest, int val, unsigned int offset,
 
        if (tx) {
                pr_debug("%s: (async) len: %zu\n", __func__, len);
-               async_tx_submit(chan, tx, flags, depend_tx, cb_fn, cb_param);
+               async_tx_submit(chan, tx, submit);
        } else { /* run the memset synchronously */
                void *dest_buf;
                pr_debug("%s: (sync) len: %zu\n", __func__, len);
 
-               dest_buf = (void *) (((char *) page_address(dest)) + offset);
+               dest_buf = page_address(dest) + offset;
 
                /* wait for any prerequisite operations */
-               async_tx_quiesce(&depend_tx);
+               async_tx_quiesce(&submit->depend_tx);
 
                memset(dest_buf, val, len);
 
-               async_tx_sync_epilog(cb_fn, cb_param);
+               async_tx_sync_epilog(submit);
        }
 
        return tx;
 }
 EXPORT_SYMBOL_GPL(async_memset);
 
-static int __init async_memset_init(void)
-{
-       return 0;
-}
-
-static void __exit async_memset_exit(void)
-{
-       do { } while (0);
-}
-
-module_init(async_memset_init);
-module_exit(async_memset_exit);
-
 MODULE_AUTHOR("Intel Corporation");
 MODULE_DESCRIPTION("asynchronous memset api");
 MODULE_LICENSE("GPL");
diff --git a/crypto/async_tx/async_pq.c b/crypto/async_tx/async_pq.c
new file mode 100644 (file)
index 0000000..b88db6d
--- /dev/null
@@ -0,0 +1,395 @@
+/*
+ * Copyright(c) 2007 Yuri Tikhonov <yur@emcraft.com>
+ * Copyright(c) 2009 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59
+ * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called COPYING.
+ */
+#include <linux/kernel.h>
+#include <linux/interrupt.h>
+#include <linux/dma-mapping.h>
+#include <linux/raid/pq.h>
+#include <linux/async_tx.h>
+
+/**
+ * scribble - space to hold throwaway P buffer for synchronous gen_syndrome
+ */
+static struct page *scribble;
+
+static bool is_raid6_zero_block(struct page *p)
+{
+       return p == (void *) raid6_empty_zero_page;
+}
+
+/* the struct page *blocks[] parameter passed to async_gen_syndrome()
+ * and async_syndrome_val() contains the 'P' destination address at
+ * blocks[disks-2] and the 'Q' destination address at blocks[disks-1]
+ *
+ * note: these are macros as they are used as lvalues
+ */
+#define P(b, d) (b[d-2])
+#define Q(b, d) (b[d-1])
+
+/**
+ * do_async_gen_syndrome - asynchronously calculate P and/or Q
+ */
+static __async_inline struct dma_async_tx_descriptor *
+do_async_gen_syndrome(struct dma_chan *chan, struct page **blocks,
+                     const unsigned char *scfs, unsigned int offset, int disks,
+                     size_t len, dma_addr_t *dma_src,
+                     struct async_submit_ctl *submit)
+{
+       struct dma_async_tx_descriptor *tx = NULL;
+       struct dma_device *dma = chan->device;
+       enum dma_ctrl_flags dma_flags = 0;
+       enum async_tx_flags flags_orig = submit->flags;
+       dma_async_tx_callback cb_fn_orig = submit->cb_fn;
+       dma_async_tx_callback cb_param_orig = submit->cb_param;
+       int src_cnt = disks - 2;
+       unsigned char coefs[src_cnt];
+       unsigned short pq_src_cnt;
+       dma_addr_t dma_dest[2];
+       int src_off = 0;
+       int idx;
+       int i;
+
+       /* DMAs use destinations as sources, so use BIDIRECTIONAL mapping */
+       if (P(blocks, disks))
+               dma_dest[0] = dma_map_page(dma->dev, P(blocks, disks), offset,
+                                          len, DMA_BIDIRECTIONAL);
+       else
+               dma_flags |= DMA_PREP_PQ_DISABLE_P;
+       if (Q(blocks, disks))
+               dma_dest[1] = dma_map_page(dma->dev, Q(blocks, disks), offset,
+                                          len, DMA_BIDIRECTIONAL);
+       else
+               dma_flags |= DMA_PREP_PQ_DISABLE_Q;
+
+       /* convert source addresses being careful to collapse 'empty'
+        * sources and update the coefficients accordingly
+        */
+       for (i = 0, idx = 0; i < src_cnt; i++) {
+               if (is_raid6_zero_block(blocks[i]))
+                       continue;
+               dma_src[idx] = dma_map_page(dma->dev, blocks[i], offset, len,
+                                           DMA_TO_DEVICE);
+               coefs[idx] = scfs[i];
+               idx++;
+       }
+       src_cnt = idx;
+
+       while (src_cnt > 0) {
+               submit->flags = flags_orig;
+               pq_src_cnt = min(src_cnt, dma_maxpq(dma, dma_flags));
+               /* if we are submitting additional pqs, leave the chain open,
+                * clear the callback parameters, and leave the destination
+                * buffers mapped
+                */
+               if (src_cnt > pq_src_cnt) {
+                       submit->flags &= ~ASYNC_TX_ACK;
+                       submit->flags |= ASYNC_TX_FENCE;
+                       dma_flags |= DMA_COMPL_SKIP_DEST_UNMAP;
+                       submit->cb_fn = NULL;
+                       submit->cb_param = NULL;
+               } else {
+                       dma_flags &= ~DMA_COMPL_SKIP_DEST_UNMAP;
+                       submit->cb_fn = cb_fn_orig;
+                       submit->cb_param = cb_param_orig;
+                       if (cb_fn_orig)
+                               dma_flags |= DMA_PREP_INTERRUPT;
+               }
+               if (submit->flags & ASYNC_TX_FENCE)
+                       dma_flags |= DMA_PREP_FENCE;
+
+               /* Since we have clobbered the src_list we are committed
+                * to doing this asynchronously.  Drivers force forward
+                * progress in case they can not provide a descriptor
+                */
+               for (;;) {
+                       tx = dma->device_prep_dma_pq(chan, dma_dest,
+                                                    &dma_src[src_off],
+                                                    pq_src_cnt,
+                                                    &coefs[src_off], len,
+                                                    dma_flags);
+                       if (likely(tx))
+                               break;
+                       async_tx_quiesce(&submit->depend_tx);
+                       dma_async_issue_pending(chan);
+               }
+
+               async_tx_submit(chan, tx, submit);
+               submit->depend_tx = tx;
+
+               /* drop completed sources */
+               src_cnt -= pq_src_cnt;
+               src_off += pq_src_cnt;
+
+               dma_flags |= DMA_PREP_CONTINUE;
+       }
+
+       return tx;
+}
+
+/**
+ * do_sync_gen_syndrome - synchronously calculate a raid6 syndrome
+ */
+static void
+do_sync_gen_syndrome(struct page **blocks, unsigned int offset, int disks,
+                    size_t len, struct async_submit_ctl *submit)
+{
+       void **srcs;
+       int i;
+
+       if (submit->scribble)
+               srcs = submit->scribble;
+       else
+               srcs = (void **) blocks;
+
+       for (i = 0; i < disks; i++) {
+               if (is_raid6_zero_block(blocks[i])) {
+                       BUG_ON(i > disks - 3); /* P or Q can't be zero */
+                       srcs[i] = blocks[i];
+               } else
+                       srcs[i] = page_address(blocks[i]) + offset;
+       }
+       raid6_call.gen_syndrome(disks, len, srcs);
+       async_tx_sync_epilog(submit);
+}
+
+/**
+ * async_gen_syndrome - asynchronously calculate a raid6 syndrome
+ * @blocks: source blocks from idx 0..disks-3, P @ disks-2 and Q @ disks-1
+ * @offset: common offset into each block (src and dest) to start transaction
+ * @disks: number of blocks (including missing P or Q, see below)
+ * @len: length of operation in bytes
+ * @submit: submission/completion modifiers
+ *
+ * General note: This routine assumes a field of GF(2^8) with a
+ * primitive polynomial of 0x11d and a generator of {02}.
+ *
+ * 'disks' note: callers can optionally omit either P or Q (but not
+ * both) from the calculation by setting blocks[disks-2] or
+ * blocks[disks-1] to NULL.  When P or Q is omitted 'len' must be <=
+ * PAGE_SIZE as a temporary buffer of this size is used in the
+ * synchronous path.  'disks' always accounts for both destination
+ * buffers.
+ *
+ * 'blocks' note: if submit->scribble is NULL then the contents of
+ * 'blocks' may be overridden
+ */
+struct dma_async_tx_descriptor *
+async_gen_syndrome(struct page **blocks, unsigned int offset, int disks,
+                  size_t len, struct async_submit_ctl *submit)
+{
+       int src_cnt = disks - 2;
+       struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
+                                                     &P(blocks, disks), 2,
+                                                     blocks, src_cnt, len);
+       struct dma_device *device = chan ? chan->device : NULL;
+       dma_addr_t *dma_src = NULL;
+
+       BUG_ON(disks > 255 || !(P(blocks, disks) || Q(blocks, disks)));
+
+       if (submit->scribble)
+               dma_src = submit->scribble;
+       else if (sizeof(dma_addr_t) <= sizeof(struct page *))
+               dma_src = (dma_addr_t *) blocks;
+
+       if (dma_src && device &&
+           (src_cnt <= dma_maxpq(device, 0) ||
+            dma_maxpq(device, DMA_PREP_CONTINUE) > 0) &&
+           is_dma_pq_aligned(device, offset, 0, len)) {
+               /* run the p+q asynchronously */
+               pr_debug("%s: (async) disks: %d len: %zu\n",
+                        __func__, disks, len);
+               return do_async_gen_syndrome(chan, blocks, raid6_gfexp, offset,
+                                            disks, len, dma_src, submit);
+       }
+
+       /* run the pq synchronously */
+       pr_debug("%s: (sync) disks: %d len: %zu\n", __func__, disks, len);
+
+       /* wait for any prerequisite operations */
+       async_tx_quiesce(&submit->depend_tx);
+
+       if (!P(blocks, disks)) {
+               P(blocks, disks) = scribble;
+               BUG_ON(len + offset > PAGE_SIZE);
+       }
+       if (!Q(blocks, disks)) {
+               Q(blocks, disks) = scribble;
+               BUG_ON(len + offset > PAGE_SIZE);
+       }
+       do_sync_gen_syndrome(blocks, offset, disks, len, submit);
+
+       return NULL;
+}
+EXPORT_SYMBOL_GPL(async_gen_syndrome);
+
+/**
+ * async_syndrome_val - asynchronously validate a raid6 syndrome
+ * @blocks: source blocks from idx 0..disks-3, P @ disks-2 and Q @ disks-1
+ * @offset: common offset into each block (src and dest) to start transaction
+ * @disks: number of blocks (including missing P or Q, see below)
+ * @len: length of operation in bytes
+ * @pqres: on val failure SUM_CHECK_P_RESULT and/or SUM_CHECK_Q_RESULT are set
+ * @spare: temporary result buffer for the synchronous case
+ * @submit: submission / completion modifiers
+ *
+ * The same notes from async_gen_syndrome apply to the 'blocks',
+ * and 'disks' parameters of this routine.  The synchronous path
+ * requires a temporary result buffer and submit->scribble to be
+ * specified.
+ */
+struct dma_async_tx_descriptor *
+async_syndrome_val(struct page **blocks, unsigned int offset, int disks,
+                  size_t len, enum sum_check_flags *pqres, struct page *spare,
+                  struct async_submit_ctl *submit)
+{
+       struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ_VAL,
+                                                     NULL, 0,  blocks, disks,
+                                                     len);
+       struct dma_device *device = chan ? chan->device : NULL;
+       struct dma_async_tx_descriptor *tx;
+       enum dma_ctrl_flags dma_flags = submit->cb_fn ? DMA_PREP_INTERRUPT : 0;
+       dma_addr_t *dma_src = NULL;
+
+       BUG_ON(disks < 4);
+
+       if (submit->scribble)
+               dma_src = submit->scribble;
+       else if (sizeof(dma_addr_t) <= sizeof(struct page *))
+               dma_src = (dma_addr_t *) blocks;
+
+       if (dma_src && device && disks <= dma_maxpq(device, 0) &&
+           is_dma_pq_aligned(device, offset, 0, len)) {
+               struct device *dev = device->dev;
+               dma_addr_t *pq = &dma_src[disks-2];
+               int i;
+
+               pr_debug("%s: (async) disks: %d len: %zu\n",
+                        __func__, disks, len);
+               if (!P(blocks, disks))
+                       dma_flags |= DMA_PREP_PQ_DISABLE_P;
+               if (!Q(blocks, disks))
+                       dma_flags |= DMA_PREP_PQ_DISABLE_Q;
+               if (submit->flags & ASYNC_TX_FENCE)
+                       dma_flags |= DMA_PREP_FENCE;
+               for (i = 0; i < disks; i++)
+                       if (likely(blocks[i])) {
+                               BUG_ON(is_raid6_zero_block(blocks[i]));
+                               dma_src[i] = dma_map_page(dev, blocks[i],
+                                                         offset, len,
+                                                         DMA_TO_DEVICE);
+                       }
+
+               for (;;) {
+                       tx = device->device_prep_dma_pq_val(chan, pq, dma_src,
+                                                           disks - 2,
+                                                           raid6_gfexp,
+                                                           len, pqres,
+                                                           dma_flags);
+                       if (likely(tx))
+                               break;
+                       async_tx_quiesce(&submit->depend_tx);
+                       dma_async_issue_pending(chan);
+               }
+               async_tx_submit(chan, tx, submit);
+
+               return tx;
+       } else {
+               struct page *p_src = P(blocks, disks);
+               struct page *q_src = Q(blocks, disks);
+               enum async_tx_flags flags_orig = submit->flags;
+               dma_async_tx_callback cb_fn_orig = submit->cb_fn;
+               void *scribble = submit->scribble;
+               void *cb_param_orig = submit->cb_param;
+               void *p, *q, *s;
+
+               pr_debug("%s: (sync) disks: %d len: %zu\n",
+                        __func__, disks, len);
+
+               /* caller must provide a temporary result buffer and
+                * allow the input parameters to be preserved
+                */
+               BUG_ON(!spare || !scribble);
+
+               /* wait for any prerequisite operations */
+               async_tx_quiesce(&submit->depend_tx);
+
+               /* recompute p and/or q into the temporary buffer and then
+                * check to see the result matches the current value
+                */
+               tx = NULL;
+               *pqres = 0;
+               if (p_src) {
+                       init_async_submit(submit, ASYNC_TX_XOR_ZERO_DST, NULL,
+                                         NULL, NULL, scribble);
+                       tx = async_xor(spare, blocks, offset, disks-2, len, submit);
+                       async_tx_quiesce(&tx);
+                       p = page_address(p_src) + offset;
+                       s = page_address(spare) + offset;
+                       *pqres |= !!memcmp(p, s, len) << SUM_CHECK_P;
+               }
+
+               if (q_src) {
+                       P(blocks, disks) = NULL;
+                       Q(blocks, disks) = spare;
+                       init_async_submit(submit, 0, NULL, NULL, NULL, scribble);
+                       tx = async_gen_syndrome(blocks, offset, disks, len, submit);
+                       async_tx_quiesce(&tx);
+                       q = page_address(q_src) + offset;
+                       s = page_address(spare) + offset;
+                       *pqres |= !!memcmp(q, s, len) << SUM_CHECK_Q;
+               }
+
+               /* restore P, Q and submit */
+               P(blocks, disks) = p_src;
+               Q(blocks, disks) = q_src;
+
+               submit->cb_fn = cb_fn_orig;
+               submit->cb_param = cb_param_orig;
+               submit->flags = flags_orig;
+               async_tx_sync_epilog(submit);
+
+               return NULL;
+       }
+}
+EXPORT_SYMBOL_GPL(async_syndrome_val);
+
+static int __init async_pq_init(void)
+{
+       scribble = alloc_page(GFP_KERNEL);
+
+       if (scribble)
+               return 0;
+
+       pr_err("%s: failed to allocate required spare page\n", __func__);
+
+       return -ENOMEM;
+}
+
+static void __exit async_pq_exit(void)
+{
+       put_page(scribble);
+}
+
+module_init(async_pq_init);
+module_exit(async_pq_exit);
+
+MODULE_DESCRIPTION("asynchronous raid6 syndrome generation/validation");
+MODULE_LICENSE("GPL");
diff --git a/crypto/async_tx/async_raid6_recov.c b/crypto/async_tx/async_raid6_recov.c
new file mode 100644 (file)
index 0000000..6d73dde
--- /dev/null
@@ -0,0 +1,468 @@
+/*
+ * Asynchronous RAID-6 recovery calculations ASYNC_TX API.
+ * Copyright(c) 2009 Intel Corporation
+ *
+ * based on raid6recov.c:
+ *   Copyright 2002 H. Peter Anvin
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 51
+ * Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ */
+#include <linux/kernel.h>
+#include <linux/interrupt.h>
+#include <linux/dma-mapping.h>
+#include <linux/raid/pq.h>
+#include <linux/async_tx.h>
+
+static struct dma_async_tx_descriptor *
+async_sum_product(struct page *dest, struct page **srcs, unsigned char *coef,
+                 size_t len, struct async_submit_ctl *submit)
+{
+       struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
+                                                     &dest, 1, srcs, 2, len);
+       struct dma_device *dma = chan ? chan->device : NULL;
+       const u8 *amul, *bmul;
+       u8 ax, bx;
+       u8 *a, *b, *c;
+
+       if (dma) {
+               dma_addr_t dma_dest[2];
+               dma_addr_t dma_src[2];
+               struct device *dev = dma->dev;
+               struct dma_async_tx_descriptor *tx;
+               enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;
+
+               if (submit->flags & ASYNC_TX_FENCE)
+                       dma_flags |= DMA_PREP_FENCE;
+               dma_dest[1] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL);
+               dma_src[0] = dma_map_page(dev, srcs[0], 0, len, DMA_TO_DEVICE);
+               dma_src[1] = dma_map_page(dev, srcs[1], 0, len, DMA_TO_DEVICE);
+               tx = dma->device_prep_dma_pq(chan, dma_dest, dma_src, 2, coef,
+                                            len, dma_flags);
+               if (tx) {
+                       async_tx_submit(chan, tx, submit);
+                       return tx;
+               }
+
+               /* could not get a descriptor, unmap and fall through to
+                * the synchronous path
+                */
+               dma_unmap_page(dev, dma_dest[1], len, DMA_BIDIRECTIONAL);
+               dma_unmap_page(dev, dma_src[0], len, DMA_TO_DEVICE);
+               dma_unmap_page(dev, dma_src[1], len, DMA_TO_DEVICE);
+       }
+
+       /* run the operation synchronously */
+       async_tx_quiesce(&submit->depend_tx);
+       amul = raid6_gfmul[coef[0]];
+       bmul = raid6_gfmul[coef[1]];
+       a = page_address(srcs[0]);
+       b = page_address(srcs[1]);
+       c = page_address(dest);
+
+       while (len--) {
+               ax    = amul[*a++];
+               bx    = bmul[*b++];
+               *c++ = ax ^ bx;
+       }
+
+       return NULL;
+}
+
+static struct dma_async_tx_descriptor *
+async_mult(struct page *dest, struct page *src, u8 coef, size_t len,
+          struct async_submit_ctl *submit)
+{
+       struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ,
+                                                     &dest, 1, &src, 1, len);
+       struct dma_device *dma = chan ? chan->device : NULL;
+       const u8 *qmul; /* Q multiplier table */
+       u8 *d, *s;
+
+       if (dma) {
+               dma_addr_t dma_dest[2];
+               dma_addr_t dma_src[1];
+               struct device *dev = dma->dev;
+               struct dma_async_tx_descriptor *tx;
+               enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P;
+
+               if (submit->flags & ASYNC_TX_FENCE)
+                       dma_flags |= DMA_PREP_FENCE;
+               dma_dest[1] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL);
+               dma_src[0] = dma_map_page(dev, src, 0, len, DMA_TO_DEVICE);
+               tx = dma->device_prep_dma_pq(chan, dma_dest, dma_src, 1, &coef,
+                                            len, dma_flags);
+               if (tx) {
+                       async_tx_submit(chan, tx, submit);
+                       return tx;
+               }
+
+               /* could not get a descriptor, unmap and fall through to
+                * the synchronous path
+                */
+               dma_unmap_page(dev, dma_dest[1], len, DMA_BIDIRECTIONAL);
+               dma_unmap_page(dev, dma_src[0], len, DMA_TO_DEVICE);
+       }
+
+       /* no channel available, or failed to allocate a descriptor, so
+        * perform the operation synchronously
+        */
+       async_tx_quiesce(&submit->depend_tx);
+       qmul  = raid6_gfmul[coef];
+       d = page_address(dest);
+       s = page_address(src);
+
+       while (len--)
+               *d++ = qmul[*s++];
+
+       return NULL;
+}
+
+static struct dma_async_tx_descriptor *
+__2data_recov_4(size_t bytes, int faila, int failb, struct page **blocks,
+             struct async_submit_ctl *submit)
+{
+       struct dma_async_tx_descriptor *tx = NULL;
+       struct page *p, *q, *a, *b;
+       struct page *srcs[2];
+       unsigned char coef[2];
+       enum async_tx_flags flags = submit->flags;
+       dma_async_tx_callback cb_fn = submit->cb_fn;
+       void *cb_param = submit->cb_param;
+       void *scribble = submit->scribble;
+
+       p = blocks[4-2];
+       q = blocks[4-1];
+
+       a = blocks[faila];
+       b = blocks[failb];
+
+       /* in the 4 disk case P + Pxy == P and Q + Qxy == Q */
+       /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
+       srcs[0] = p;
+       srcs[1] = q;
+       coef[0] = raid6_gfexi[failb-faila];
+       coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
+       init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
+       tx = async_sum_product(b, srcs, coef, bytes, submit);
+
+       /* Dy = P+Pxy+Dx */
+       srcs[0] = p;
+       srcs[1] = b;
+       init_async_submit(submit, flags | ASYNC_TX_XOR_ZERO_DST, tx, cb_fn,
+                         cb_param, scribble);
+       tx = async_xor(a, srcs, 0, 2, bytes, submit);
+
+       return tx;
+
+}
+
+static struct dma_async_tx_descriptor *
+__2data_recov_5(size_t bytes, int faila, int failb, struct page **blocks,
+             struct async_submit_ctl *submit)
+{
+       struct dma_async_tx_descriptor *tx = NULL;
+       struct page *p, *q, *g, *dp, *dq;
+       struct page *srcs[2];
+       unsigned char coef[2];
+       enum async_tx_flags flags = submit->flags;
+       dma_async_tx_callback cb_fn = submit->cb_fn;
+       void *cb_param = submit->cb_param;
+       void *scribble = submit->scribble;
+       int uninitialized_var(good);
+       int i;
+
+       for (i = 0; i < 3; i++) {
+               if (i == faila || i == failb)
+                       continue;
+               else {
+                       good = i;
+                       break;
+               }
+       }
+       BUG_ON(i >= 3);
+
+       p = blocks[5-2];
+       q = blocks[5-1];
+       g = blocks[good];
+
+       /* Compute syndrome with zero for the missing data pages
+        * Use the dead data pages as temporary storage for delta p and
+        * delta q
+        */
+       dp = blocks[faila];
+       dq = blocks[failb];
+
+       init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
+       tx = async_memcpy(dp, g, 0, 0, bytes, submit);
+       init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
+       tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit);
+
+       /* compute P + Pxy */
+       srcs[0] = dp;
+       srcs[1] = p;
+       init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
+                         NULL, NULL, scribble);
+       tx = async_xor(dp, srcs, 0, 2, bytes, submit);
+
+       /* compute Q + Qxy */
+       srcs[0] = dq;
+       srcs[1] = q;
+       init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
+                         NULL, NULL, scribble);
+       tx = async_xor(dq, srcs, 0, 2, bytes, submit);
+
+       /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
+       srcs[0] = dp;
+       srcs[1] = dq;
+       coef[0] = raid6_gfexi[failb-faila];
+       coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
+       init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
+       tx = async_sum_product(dq, srcs, coef, bytes, submit);
+
+       /* Dy = P+Pxy+Dx */
+       srcs[0] = dp;
+       srcs[1] = dq;
+       init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
+                         cb_param, scribble);
+       tx = async_xor(dp, srcs, 0, 2, bytes, submit);
+
+       return tx;
+}
+
+static struct dma_async_tx_descriptor *
+__2data_recov_n(int disks, size_t bytes, int faila, int failb,
+             struct page **blocks, struct async_submit_ctl *submit)
+{
+       struct dma_async_tx_descriptor *tx = NULL;
+       struct page *p, *q, *dp, *dq;
+       struct page *srcs[2];
+       unsigned char coef[2];
+       enum async_tx_flags flags = submit->flags;
+       dma_async_tx_callback cb_fn = submit->cb_fn;
+       void *cb_param = submit->cb_param;
+       void *scribble = submit->scribble;
+
+       p = blocks[disks-2];
+       q = blocks[disks-1];
+
+       /* Compute syndrome with zero for the missing data pages
+        * Use the dead data pages as temporary storage for
+        * delta p and delta q
+        */
+       dp = blocks[faila];
+       blocks[faila] = (void *)raid6_empty_zero_page;
+       blocks[disks-2] = dp;
+       dq = blocks[failb];
+       blocks[failb] = (void *)raid6_empty_zero_page;
+       blocks[disks-1] = dq;
+
+       init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
+       tx = async_gen_syndrome(blocks, 0, disks, bytes, submit);
+
+       /* Restore pointer table */
+       blocks[faila]   = dp;
+       blocks[failb]   = dq;
+       blocks[disks-2] = p;
+       blocks[disks-1] = q;
+
+       /* compute P + Pxy */
+       srcs[0] = dp;
+       srcs[1] = p;
+       init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
+                         NULL, NULL, scribble);
+       tx = async_xor(dp, srcs, 0, 2, bytes, submit);
+
+       /* compute Q + Qxy */
+       srcs[0] = dq;
+       srcs[1] = q;
+       init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
+                         NULL, NULL, scribble);
+       tx = async_xor(dq, srcs, 0, 2, bytes, submit);
+
+       /* Dx = A*(P+Pxy) + B*(Q+Qxy) */
+       srcs[0] = dp;
+       srcs[1] = dq;
+       coef[0] = raid6_gfexi[failb-faila];
+       coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]];
+       init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
+       tx = async_sum_product(dq, srcs, coef, bytes, submit);
+
+       /* Dy = P+Pxy+Dx */
+       srcs[0] = dp;
+       srcs[1] = dq;
+       init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
+                         cb_param, scribble);
+       tx = async_xor(dp, srcs, 0, 2, bytes, submit);
+
+       return tx;
+}
+
+/**
+ * async_raid6_2data_recov - asynchronously calculate two missing data blocks
+ * @disks: number of disks in the RAID-6 array
+ * @bytes: block size
+ * @faila: first failed drive index
+ * @failb: second failed drive index
+ * @blocks: array of source pointers where the last two entries are p and q
+ * @submit: submission/completion modifiers
+ */
+struct dma_async_tx_descriptor *
+async_raid6_2data_recov(int disks, size_t bytes, int faila, int failb,
+                       struct page **blocks, struct async_submit_ctl *submit)
+{
+       BUG_ON(faila == failb);
+       if (failb < faila)
+               swap(faila, failb);
+
+       pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes);
+
+       /* we need to preserve the contents of 'blocks' for the async
+        * case, so punt to synchronous if a scribble buffer is not available
+        */
+       if (!submit->scribble) {
+               void **ptrs = (void **) blocks;
+               int i;
+
+               async_tx_quiesce(&submit->depend_tx);
+               for (i = 0; i < disks; i++)
+                       ptrs[i] = page_address(blocks[i]);
+
+               raid6_2data_recov(disks, bytes, faila, failb, ptrs);
+
+               async_tx_sync_epilog(submit);
+
+               return NULL;
+       }
+
+       switch (disks) {
+       case 4:
+               /* dma devices do not uniformly understand a zero source pq
+                * operation (in contrast to the synchronous case), so
+                * explicitly handle the 4 disk special case
+                */
+               return __2data_recov_4(bytes, faila, failb, blocks, submit);
+       case 5:
+               /* dma devices do not uniformly understand a single
+                * source pq operation (in contrast to the synchronous
+                * case), so explicitly handle the 5 disk special case
+                */
+               return __2data_recov_5(bytes, faila, failb, blocks, submit);
+       default:
+               return __2data_recov_n(disks, bytes, faila, failb, blocks, submit);
+       }
+}
+EXPORT_SYMBOL_GPL(async_raid6_2data_recov);
+
+/**
+ * async_raid6_datap_recov - asynchronously calculate a data and the 'p' block
+ * @disks: number of disks in the RAID-6 array
+ * @bytes: block size
+ * @faila: failed drive index
+ * @blocks: array of source pointers where the last two entries are p and q
+ * @submit: submission/completion modifiers
+ */
+struct dma_async_tx_descriptor *
+async_raid6_datap_recov(int disks, size_t bytes, int faila,
+                       struct page **blocks, struct async_submit_ctl *submit)
+{
+       struct dma_async_tx_descriptor *tx = NULL;
+       struct page *p, *q, *dq;
+       u8 coef;
+       enum async_tx_flags flags = submit->flags;
+       dma_async_tx_callback cb_fn = submit->cb_fn;
+       void *cb_param = submit->cb_param;
+       void *scribble = submit->scribble;
+       struct page *srcs[2];
+
+       pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes);
+
+       /* we need to preserve the contents of 'blocks' for the async
+        * case, so punt to synchronous if a scribble buffer is not available
+        */
+       if (!scribble) {
+               void **ptrs = (void **) blocks;
+               int i;
+
+               async_tx_quiesce(&submit->depend_tx);
+               for (i = 0; i < disks; i++)
+                       ptrs[i] = page_address(blocks[i]);
+
+               raid6_datap_recov(disks, bytes, faila, ptrs);
+
+               async_tx_sync_epilog(submit);
+
+               return NULL;
+       }
+
+       p = blocks[disks-2];
+       q = blocks[disks-1];
+
+       /* Compute syndrome with zero for the missing data page
+        * Use the dead data page as temporary storage for delta q
+        */
+       dq = blocks[faila];
+       blocks[faila] = (void *)raid6_empty_zero_page;
+       blocks[disks-1] = dq;
+
+       /* in the 4 disk case we only need to perform a single source
+        * multiplication
+        */
+       if (disks == 4) {
+               int good = faila == 0 ? 1 : 0;
+               struct page *g = blocks[good];
+
+               init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
+                                 scribble);
+               tx = async_memcpy(p, g, 0, 0, bytes, submit);
+
+               init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
+                                 scribble);
+               tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit);
+       } else {
+               init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL,
+                                 scribble);
+               tx = async_gen_syndrome(blocks, 0, disks, bytes, submit);
+       }
+
+       /* Restore pointer table */
+       blocks[faila]   = dq;
+       blocks[disks-1] = q;
+
+       /* calculate g^{-faila} */
+       coef = raid6_gfinv[raid6_gfexp[faila]];
+
+       srcs[0] = dq;
+       srcs[1] = q;
+       init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
+                         NULL, NULL, scribble);
+       tx = async_xor(dq, srcs, 0, 2, bytes, submit);
+
+       init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble);
+       tx = async_mult(dq, dq, coef, bytes, submit);
+
+       srcs[0] = p;
+       srcs[1] = dq;
+       init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn,
+                         cb_param, scribble);
+       tx = async_xor(p, srcs, 0, 2, bytes, submit);
+
+       return tx;
+}
+EXPORT_SYMBOL_GPL(async_raid6_datap_recov);
+
+MODULE_AUTHOR("Dan Williams <dan.j.williams@intel.com>");
+MODULE_DESCRIPTION("asynchronous RAID-6 recovery api");
+MODULE_LICENSE("GPL");
index 06eb6cc..f9cdf04 100644 (file)
@@ -42,16 +42,21 @@ static void __exit async_tx_exit(void)
        async_dmaengine_put();
 }
 
+module_init(async_tx_init);
+module_exit(async_tx_exit);
+
 /**
  * __async_tx_find_channel - find a channel to carry out the operation or let
  *     the transaction execute synchronously
- * @depend_tx: transaction dependency
+ * @submit: transaction dependency and submission modifiers
  * @tx_type: transaction type
  */
 struct dma_chan *
-__async_tx_find_channel(struct dma_async_tx_descriptor *depend_tx,
-       enum dma_transaction_type tx_type)
+__async_tx_find_channel(struct async_submit_ctl *submit,
+                       enum dma_transaction_type tx_type)
 {
+       struct dma_async_tx_descriptor *depend_tx = submit->depend_tx;
+
        /* see if we can keep the chain on one channel */
        if (depend_tx &&
            dma_has_cap(tx_type, depend_tx->chan->device->cap_mask))
@@ -59,17 +64,6 @@ __async_tx_find_channel(struct dma_async_tx_descriptor *depend_tx,
        return async_dma_find_channel(tx_type);
 }
 EXPORT_SYMBOL_GPL(__async_tx_find_channel);
-#else
-static int __init async_tx_init(void)
-{
-       printk(KERN_INFO "async_tx: api initialized (sync-only)\n");
-       return 0;
-}
-
-static void __exit async_tx_exit(void)
-{
-       do { } while (0);
-}
 #endif
 
 
@@ -83,10 +77,14 @@ static void
 async_tx_channel_switch(struct dma_async_tx_descriptor *depend_tx,
                        struct dma_async_tx_descriptor *tx)
 {
-       struct dma_chan *chan;
-       struct dma_device *device;
+       struct dma_chan *chan = depend_tx->chan;
+       struct dma_device *device = chan->device;
        struct dma_async_tx_descriptor *intr_tx = (void *) ~0;
 
+       #ifdef CONFIG_ASYNC_TX_DISABLE_CHANNEL_SWITCH
+       BUG();
+       #endif
+
        /* first check to see if we can still append to depend_tx */
        spin_lock_bh(&depend_tx->lock);
        if (depend_tx->parent && depend_tx->chan == tx->chan) {
@@ -96,11 +94,11 @@ async_tx_channel_switch(struct dma_async_tx_descriptor *depend_tx,
        }
        spin_unlock_bh(&depend_tx->lock);
 
-       if (!intr_tx)
+       /* attached dependency, flush the parent channel */
+       if (!intr_tx) {
+               device->device_issue_pending(chan);
                return;
-
-       chan = depend_tx->chan;
-       device = chan->device;
+       }
 
        /* see if we can schedule an interrupt
         * otherwise poll for completion
@@ -134,6 +132,7 @@ async_tx_channel_switch(struct dma_async_tx_descriptor *depend_tx,
                        intr_tx->tx_submit(intr_tx);
                        async_tx_ack(intr_tx);
                }
+               device->device_issue_pending(chan);
        } else {
                if (dma_wait_for_async_tx(depend_tx) == DMA_ERROR)
                        panic("%s: DMA_ERROR waiting for depend_tx\n",
@@ -144,13 +143,14 @@ async_tx_channel_switch(struct dma_async_tx_descriptor *depend_tx,
 
 
 /**
- * submit_disposition - while holding depend_tx->lock we must avoid submitting
- *     new operations to prevent a circular locking dependency with
- *     drivers that already hold a channel lock when calling
- *     async_tx_run_dependencies.
+ * submit_disposition - flags for routing an incoming operation
  * @ASYNC_TX_SUBMITTED: we were able to append the new operation under the lock
  * @ASYNC_TX_CHANNEL_SWITCH: when the lock is dropped schedule a channel switch
  * @ASYNC_TX_DIRECT_SUBMIT: when the lock is dropped submit directly
+ *
+ * while holding depend_tx->lock we must avoid submitting new operations
+ * to prevent a circular locking dependency with drivers that already
+ * hold a channel lock when calling async_tx_run_dependencies.
  */
 enum submit_disposition {
        ASYNC_TX_SUBMITTED,
@@ -160,11 +160,12 @@ enum submit_disposition {
 
 void
 async_tx_submit(struct dma_chan *chan, struct dma_async_tx_descriptor *tx,
-       enum async_tx_flags flags, struct dma_async_tx_descriptor *depend_tx,
-       dma_async_tx_callback cb_fn, void *cb_param)
+               struct async_submit_ctl *submit)
 {
-       tx->callback = cb_fn;
-       tx->callback_param = cb_param;
+       struct dma_async_tx_descriptor *depend_tx = submit->depend_tx;
+
+       tx->callback = submit->cb_fn;
+       tx->callback_param = submit->cb_param;
 
        if (depend_tx) {
                enum submit_disposition s;
@@ -220,30 +221,29 @@ async_tx_submit(struct dma_chan *chan, struct dma_async_tx_descriptor *tx,
                tx->tx_submit(tx);
        }
 
-       if (flags & ASYNC_TX_ACK)
+       if (submit->flags & ASYNC_TX_ACK)
                async_tx_ack(tx);
 
-       if (depend_tx && (flags & ASYNC_TX_DEP_ACK))
+       if (depend_tx)
                async_tx_ack(depend_tx);
 }
 EXPORT_SYMBOL_GPL(async_tx_submit);
 
 /**
- * async_trigger_callback - schedules the callback function to be run after
- * any dependent operations have been completed.
- * @flags: ASYNC_TX_ACK, ASYNC_TX_DEP_ACK
- * @depend_tx: 'callback' requires the completion of this transaction
- * @cb_fn: function to call after depend_tx completes
- * @cb_param: parameter to pass to the callback routine
+ * async_trigger_callback - schedules the callback function to be run
+ * @submit: submission and completion parameters
+ *
+ * honored flags: ASYNC_TX_ACK
+ *
+ * The callback is run after any dependent operations have completed.
  */
 struct dma_async_tx_descriptor *
-async_trigger_callback(enum async_tx_flags flags,
-       struct dma_async_tx_descriptor *depend_tx,
-       dma_async_tx_callback cb_fn, void *cb_param)
+async_trigger_callback(struct async_submit_ctl *submit)
 {
        struct dma_chan *chan;
        struct dma_device *device;
        struct dma_async_tx_descriptor *tx;
+       struct dma_async_tx_descriptor *depend_tx = submit->depend_tx;
 
        if (depend_tx) {
                chan = depend_tx->chan;
@@ -262,14 +262,14 @@ async_trigger_callback(enum async_tx_flags flags,
        if (tx) {
                pr_debug("%s: (async)\n", __func__);
 
-               async_tx_submit(chan, tx, flags, depend_tx, cb_fn, cb_param);
+               async_tx_submit(chan, tx, submit);
        } else {
                pr_debug("%s: (sync)\n", __func__);
 
                /* wait for any prerequisite operations */
-               async_tx_quiesce(&depend_tx);
+               async_tx_quiesce(&submit->depend_tx);
 
-               async_tx_sync_epilog(cb_fn, cb_param);
+               async_tx_sync_epilog(submit);
        }
 
        return tx;
@@ -295,9 +295,6 @@ void async_tx_quiesce(struct dma_async_tx_descriptor **tx)
 }
 EXPORT_SYMBOL_GPL(async_tx_quiesce);
 
-module_init(async_tx_init);
-module_exit(async_tx_exit);
-
 MODULE_AUTHOR("Intel Corporation");
 MODULE_DESCRIPTION("Asynchronous Bulk Memory Transactions API");
 MODULE_LICENSE("GPL");
index 90dd3f8..b459a90 100644 (file)
 /* do_async_xor - dma map the pages and perform the xor with an engine */
 static __async_inline struct dma_async_tx_descriptor *
 do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list,
-            unsigned int offset, int src_cnt, size_t len,
-            enum async_tx_flags flags,
-            struct dma_async_tx_descriptor *depend_tx,
-            dma_async_tx_callback cb_fn, void *cb_param)
+            unsigned int offset, int src_cnt, size_t len, dma_addr_t *dma_src,
+            struct async_submit_ctl *submit)
 {
        struct dma_device *dma = chan->device;
-       dma_addr_t *dma_src = (dma_addr_t *) src_list;
        struct dma_async_tx_descriptor *tx = NULL;
        int src_off = 0;
        int i;
-       dma_async_tx_callback _cb_fn;
-       void *_cb_param;
-       enum async_tx_flags async_flags;
+       dma_async_tx_callback cb_fn_orig = submit->cb_fn;
+       void *cb_param_orig = submit->cb_param;
+       enum async_tx_flags flags_orig = submit->flags;
        enum dma_ctrl_flags dma_flags;
        int xor_src_cnt;
        dma_addr_t dma_dest;
@@ -63,25 +60,27 @@ do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list,
        }
 
        while (src_cnt) {
-               async_flags = flags;
+               submit->flags = flags_orig;
                dma_flags = 0;
-               xor_src_cnt = min(src_cnt, dma->max_xor);
+               xor_src_cnt = min(src_cnt, (int)dma->max_xor);
                /* if we are submitting additional xors, leave the chain open,
                 * clear the callback parameters, and leave the destination
                 * buffer mapped
                 */
                if (src_cnt > xor_src_cnt) {
-                       async_flags &= ~ASYNC_TX_ACK;
+                       submit->flags &= ~ASYNC_TX_ACK;
+                       submit->flags |= ASYNC_TX_FENCE;
                        dma_flags = DMA_COMPL_SKIP_DEST_UNMAP;
-                       _cb_fn = NULL;
-                       _cb_param = NULL;
+                       submit->cb_fn = NULL;
+                       submit->cb_param = NULL;
                } else {
-                       _cb_fn = cb_fn;
-                       _cb_param = cb_param;
+                       submit->cb_fn = cb_fn_orig;
+                       submit->cb_param = cb_param_orig;
                }
-               if (_cb_fn)
+               if (submit->cb_fn)
                        dma_flags |= DMA_PREP_INTERRUPT;
-
+               if (submit->flags & ASYNC_TX_FENCE)
+                       dma_flags |= DMA_PREP_FENCE;
                /* Since we have clobbered the src_list we are committed
                 * to doing this asynchronously.  Drivers force forward progress
                 * in case they can not provide a descriptor
@@ -90,7 +89,7 @@ do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list,
                                              xor_src_cnt, len, dma_flags);
 
                if (unlikely(!tx))
-                       async_tx_quiesce(&depend_tx);
+                       async_tx_quiesce(&submit->depend_tx);
 
                /* spin wait for the preceeding transactions to complete */
                while (unlikely(!tx)) {
@@ -101,11 +100,8 @@ do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list,
                                                      dma_flags);
                }
 
-               async_tx_submit(chan, tx, async_flags, depend_tx, _cb_fn,
-                               _cb_param);
-
-               depend_tx = tx;
-               flags |= ASYNC_TX_DEP_ACK;
+               async_tx_submit(chan, tx, submit);
+               submit->depend_tx = tx;
 
                if (src_cnt > xor_src_cnt) {
                        /* drop completed sources */
@@ -124,23 +120,27 @@ do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list,
 
 static void
 do_sync_xor(struct page *dest, struct page **src_list, unsigned int offset,
-           int src_cnt, size_t len, enum async_tx_flags flags,
-           dma_async_tx_callback cb_fn, void *cb_param)
+           int src_cnt, size_t len, struct async_submit_ctl *submit)
 {
        int i;
        int xor_src_cnt;
        int src_off = 0;
        void *dest_buf;
-       void **srcs = (void **) src_list;
+       void **srcs;
+
+       if (submit->scribble)
+               srcs = submit->scribble;
+       else
+               srcs = (void **) src_list;
 
-       /* reuse the 'src_list' array to convert to buffer pointers */
+       /* convert to buffer pointers */
        for (i = 0; i < src_cnt; i++)
                srcs[i] = page_address(src_list[i]) + offset;
 
        /* set destination address */
        dest_buf = page_address(dest) + offset;
 
-       if (flags & ASYNC_TX_XOR_ZERO_DST)
+       if (submit->flags & ASYNC_TX_XOR_ZERO_DST)
                memset(dest_buf, 0, len);
 
        while (src_cnt > 0) {
@@ -153,61 +153,70 @@ do_sync_xor(struct page *dest, struct page **src_list, unsigned int offset,
                src_off += xor_src_cnt;
        }
 
-       async_tx_sync_epilog(cb_fn, cb_param);
+       async_tx_sync_epilog(submit);
 }
 
 /**
  * async_xor - attempt to xor a set of blocks with a dma engine.
- *     xor_blocks always uses the dest as a source so the ASYNC_TX_XOR_ZERO_DST
- *     flag must be set to not include dest data in the calculation.  The
- *     assumption with dma eninges is that they only use the destination
- *     buffer as a source when it is explicity specified in the source list.
  * @dest: destination page
- * @src_list: array of source pages (if the dest is also a source it must be
- *     at index zero).  The contents of this array may be overwritten.
- * @offset: offset in pages to start transaction
+ * @src_list: array of source pages
+ * @offset: common src/dst offset to start transaction
  * @src_cnt: number of source pages
  * @len: length in bytes
- * @flags: ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DEST,
- *     ASYNC_TX_ACK, ASYNC_TX_DEP_ACK
- * @depend_tx: xor depends on the result of this transaction.
- * @cb_fn: function to call when the xor completes
- * @cb_param: parameter to pass to the callback routine
+ * @submit: submission / completion modifiers
+ *
+ * honored flags: ASYNC_TX_ACK, ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DST
+ *
+ * xor_blocks always uses the dest as a source so the
+ * ASYNC_TX_XOR_ZERO_DST flag must be set to not include dest data in
+ * the calculation.  The assumption with dma eninges is that they only
+ * use the destination buffer as a source when it is explicity specified
+ * in the source list.
+ *
+ * src_list note: if the dest is also a source it must be at index zero.
+ * The contents of this array will be overwritten if a scribble region
+ * is not specified.
  */
 struct dma_async_tx_descriptor *
 async_xor(struct page *dest, struct page **src_list, unsigned int offset,
-       int src_cnt, size_t len, enum async_tx_flags flags,
-       struct dma_async_tx_descriptor *depend_tx,
-       dma_async_tx_callback cb_fn, void *cb_param)
+         int src_cnt, size_t len, struct async_submit_ctl *submit)
 {
-       struct dma_chan *chan = async_tx_find_channel(depend_tx, DMA_XOR,
+       struct dma_chan *chan = async_tx_find_channel(submit, DMA_XOR,
                                                      &dest, 1, src_list,
                                                      src_cnt, len);
+       dma_addr_t *dma_src = NULL;
+
        BUG_ON(src_cnt <= 1);
 
-       if (chan) {
+       if (submit->scribble)
+               dma_src = submit->scribble;
+       else if (sizeof(dma_addr_t) <= sizeof(struct page *))
+               dma_src = (dma_addr_t *) src_list;
+
+       if (dma_src && chan && is_dma_xor_aligned(chan->device, offset, 0, len)) {
                /* run the xor asynchronously */
                pr_debug("%s (async): len: %zu\n", __func__, len);
 
                return do_async_xor(chan, dest, src_list, offset, src_cnt, len,
-                                   flags, depend_tx, cb_fn, cb_param);
+                                   dma_src, submit);
        } else {
                /* run the xor synchronously */
                pr_debug("%s (sync): len: %zu\n", __func__, len);
+               WARN_ONCE(chan, "%s: no space for dma address conversion\n",
+                         __func__);
 
                /* in the sync case the dest is an implied source
                 * (assumes the dest is the first source)
                 */
-               if (flags & ASYNC_TX_XOR_DROP_DST) {
+               if (submit->flags & ASYNC_TX_XOR_DROP_DST) {
                        src_cnt--;
                        src_list++;
                }
 
                /* wait for any prerequisite operations */
-               async_tx_quiesce(&depend_tx);
+               async_tx_quiesce(&submit->depend_tx);
 
-               do_sync_xor(dest, src_list, offset, src_cnt, len,
-                           flags, cb_fn, cb_param);
+               do_sync_xor(dest, src_list, offset, src_cnt, len, submit);
 
                return NULL;
        }
@@ -222,104 +231,94 @@ static int page_is_zero(struct page *p, unsigned int offset, size_t len)
 }
 
 /**
- * async_xor_zero_sum - attempt a xor parity check with a dma engine.
+ * async_xor_val - attempt a xor parity check with a dma engine.
  * @dest: destination page used if the xor is performed synchronously
- * @src_list: array of source pages.  The dest page must be listed as a source
- *     at index zero.  The contents of this array may be overwritten.
+ * @src_list: array of source pages
  * @offset: offset in pages to start transaction
  * @src_cnt: number of source pages
  * @len: length in bytes
  * @result: 0 if sum == 0 else non-zero
- * @flags: ASYNC_TX_ACK, ASYNC_TX_DEP_ACK
- * @depend_tx: xor depends on the result of this transaction.
- * @cb_fn: function to call when the xor completes
- * @cb_param: parameter to pass to the callback routine
+ * @submit: submission / completion modifiers
+ *
+ * honored flags: ASYNC_TX_ACK
+ *
+ * src_list note: if the dest is also a source it must be at index zero.
+ * The contents of this array will be overwritten if a scribble region
+ * is not specified.
  */
 struct dma_async_tx_descriptor *
-async_xor_zero_sum(struct page *dest, struct page **src_list,
-       unsigned int offset, int src_cnt, size_t len,
-       u32 *result, enum async_tx_flags flags,
-       struct dma_async_tx_descriptor *depend_tx,
-       dma_async_tx_callback cb_fn, void *cb_param)
+async_xor_val(struct page *dest, struct page **src_list, unsigned int offset,
+             int src_cnt, size_t len, enum sum_check_flags *result,
+             struct async_submit_ctl *submit)
 {
-       struct dma_chan *chan = async_tx_find_channel(depend_tx, DMA_ZERO_SUM,
+       struct dma_chan *chan = async_tx_find_channel(submit, DMA_XOR_VAL,
                                                      &dest, 1, src_list,
                                                      src_cnt, len);
        struct dma_device *device = chan ? chan->device : NULL;
        struct dma_async_tx_descriptor *tx = NULL;
+       dma_addr_t *dma_src = NULL;
 
        BUG_ON(src_cnt <= 1);
 
-       if (device && src_cnt <= device->max_xor) {
-               dma_addr_t *dma_src = (dma_addr_t *) src_list;
-               unsigned long dma_prep_flags = cb_fn ? DMA_PREP_INTERRUPT : 0;
+       if (submit->scribble)
+               dma_src = submit->scribble;
+       else if (sizeof(dma_addr_t) <= sizeof(struct page *))
+               dma_src = (dma_addr_t *) src_list;
+
+       if (dma_src && device && src_cnt <= device->max_xor &&
+           is_dma_xor_aligned(device, offset, 0, len)) {
+               unsigned long dma_prep_flags = 0;
                int i;
 
                pr_debug("%s: (async) len: %zu\n", __func__, len);
 
+               if (submit->cb_fn)
+                       dma_prep_flags |= DMA_PREP_INTERRUPT;
+               if (submit->flags & ASYNC_TX_FENCE)
+                       dma_prep_flags |= DMA_PREP_FENCE;
                for (i = 0; i < src_cnt; i++)
                        dma_src[i] = dma_map_page(device->dev, src_list[i],
                                                  offset, len, DMA_TO_DEVICE);
 
-               tx = device->device_prep_dma_zero_sum(chan, dma_src, src_cnt,
-                                                     len, result,
-                                                     dma_prep_flags);
+               tx = device->device_prep_dma_xor_val(chan, dma_src, src_cnt,
+                                                    len, result,
+                                                    dma_prep_flags);
                if (unlikely(!tx)) {
-                       async_tx_quiesce(&depend_tx);
+                       async_tx_quiesce(&submit->depend_tx);
 
                        while (!tx) {
                                dma_async_issue_pending(chan);
-                               tx = device->device_prep_dma_zero_sum(chan,
+                               tx = device->device_prep_dma_xor_val(chan,
                                        dma_src, src_cnt, len, result,
                                        dma_prep_flags);
                        }
                }
 
-               async_tx_submit(chan, tx, flags, depend_tx, cb_fn, cb_param);
+               async_tx_submit(chan, tx, submit);
        } else {
-               unsigned long xor_flags = flags;
+               enum async_tx_flags flags_orig = submit->flags;
 
                pr_debug("%s: (sync) len: %zu\n", __func__, len);
+               WARN_ONCE(device && src_cnt <= device->max_xor,
+                         "%s: no space for dma address conversion\n",
+                         __func__);
 
-               xor_flags |= ASYNC_TX_XOR_DROP_DST;
-               xor_flags &= ~ASYNC_TX_ACK;
+               submit->flags |= ASYNC_TX_XOR_DROP_DST;
+               submit->flags &= ~ASYNC_TX_ACK;
 
-               tx = async_xor(dest, src_list, offset, src_cnt, len, xor_flags,
-                       depend_tx, NULL, NULL);
+               tx = async_xor(dest, src_list, offset, src_cnt, len, submit);
 
                async_tx_quiesce(&tx);
 
-               *result = page_is_zero(dest, offset, len) ? 0 : 1;
+               *result = !page_is_zero(dest, offset, len) << SUM_CHECK_P;
 
-               async_tx_sync_epilog(cb_fn, cb_param);
+               async_tx_sync_epilog(submit);
+               submit->flags = flags_orig;
        }
 
        return tx;
 }
-EXPORT_SYMBOL_GPL(async_xor_zero_sum);
-
-static int __init async_xor_init(void)
-{
-       #ifdef CONFIG_ASYNC_TX_DMA
-       /* To conserve stack space the input src_list (array of page pointers)
-        * is reused to hold the array of dma addresses passed to the driver.
-        * This conversion is only possible when dma_addr_t is less than the
-        * the size of a pointer.  HIGHMEM64G is known to violate this
-        * assumption.
-        */
-       BUILD_BUG_ON(sizeof(dma_addr_t) > sizeof(struct page *));
-       #endif
-
-       return 0;
-}
-
-static void __exit async_xor_exit(void)
-{
-       do { } while (0);
-}
-
-module_init(async_xor_init);
-module_exit(async_xor_exit);
+EXPORT_SYMBOL_GPL(async_xor_val);
 
 MODULE_AUTHOR("Intel Corporation");
 MODULE_DESCRIPTION("asynchronous xor/xor-zero-sum api");
diff --git a/crypto/async_tx/raid6test.c b/crypto/async_tx/raid6test.c
new file mode 100644 (file)
index 0000000..3ec27c7
--- /dev/null
@@ -0,0 +1,240 @@
+/*
+ * asynchronous raid6 recovery self test
+ * Copyright (c) 2009, Intel Corporation.
+ *
+ * based on drivers/md/raid6test/test.c:
+ *     Copyright 2002-2007 H. Peter Anvin
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ */
+#include <linux/async_tx.h>
+#include <linux/random.h>
+
+#undef pr
+#define pr(fmt, args...) pr_info("raid6test: " fmt, ##args)
+
+#define NDISKS 16 /* Including P and Q */
+
+static struct page *dataptrs[NDISKS];
+static addr_conv_t addr_conv[NDISKS];
+static struct page *data[NDISKS+3];
+static struct page *spare;
+static struct page *recovi;
+static struct page *recovj;
+
+static void callback(void *param)
+{
+       struct completion *cmp = param;
+
+       complete(cmp);
+}
+
+static void makedata(int disks)
+{
+       int i, j;
+
+       for (i = 0; i < disks; i++) {
+               for (j = 0; j < PAGE_SIZE/sizeof(u32); j += sizeof(u32)) {
+                       u32 *p = page_address(data[i]) + j;
+
+                       *p = random32();
+               }
+
+               dataptrs[i] = data[i];
+       }
+}
+
+static char disk_type(int d, int disks)
+{
+       if (d == disks - 2)
+               return 'P';
+       else if (d == disks - 1)
+               return 'Q';
+       else
+               return 'D';
+}
+
+/* Recover two failed blocks. */
+static void raid6_dual_recov(int disks, size_t bytes, int faila, int failb, struct page **ptrs)
+{
+       struct async_submit_ctl submit;
+       struct completion cmp;
+       struct dma_async_tx_descriptor *tx = NULL;
+       enum sum_check_flags result = ~0;
+
+       if (faila > failb)
+               swap(faila, failb);
+
+       if (failb == disks-1) {
+               if (faila == disks-2) {
+                       /* P+Q failure.  Just rebuild the syndrome. */
+                       init_async_submit(&submit, 0, NULL, NULL, NULL, addr_conv);
+                       tx = async_gen_syndrome(ptrs, 0, disks, bytes, &submit);
+               } else {
+                       struct page *blocks[disks];
+                       struct page *dest;
+                       int count = 0;
+                       int i;
+
+                       /* data+Q failure.  Reconstruct data from P,
+                        * then rebuild syndrome
+                        */
+                       for (i = disks; i-- ; ) {
+                               if (i == faila || i == failb)
+                                       continue;
+                               blocks[count++] = ptrs[i];
+                       }
+                       dest = ptrs[faila];
+                       init_async_submit(&submit, ASYNC_TX_XOR_ZERO_DST, NULL,
+                                         NULL, NULL, addr_conv);
+                       tx = async_xor(dest, blocks, 0, count, bytes, &submit);
+
+                       init_async_submit(&submit, 0, tx, NULL, NULL, addr_conv);
+                       tx = async_gen_syndrome(ptrs, 0, disks, bytes, &submit);
+               }
+       } else {
+               if (failb == disks-2) {
+                       /* data+P failure. */
+                       init_async_submit(&submit, 0, NULL, NULL, NULL, addr_conv);
+                       tx = async_raid6_datap_recov(disks, bytes, faila, ptrs, &submit);
+               } else {
+                       /* data+data failure. */
+                       init_async_submit(&submit, 0, NULL, NULL, NULL, addr_conv);
+                       tx = async_raid6_2data_recov(disks, bytes, faila, failb, ptrs, &submit);
+               }
+       }
+       init_completion(&cmp);
+       init_async_submit(&submit, ASYNC_TX_ACK, tx, callback, &cmp, addr_conv);
+       tx = async_syndrome_val(ptrs, 0, disks, bytes, &result, spare, &submit);
+       async_tx_issue_pending(tx);
+
+       if (wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)) == 0)
+               pr("%s: timeout! (faila: %d failb: %d disks: %d)\n",
+                  __func__, faila, failb, disks);
+
+       if (result != 0)
+               pr("%s: validation failure! faila: %d failb: %d sum_check_flags: %x\n",
+                  __func__, faila, failb, result);
+}
+
+static int test_disks(int i, int j, int disks)
+{
+       int erra, errb;
+
+       memset(page_address(recovi), 0xf0, PAGE_SIZE);
+       memset(page_address(recovj), 0xba, PAGE_SIZE);
+
+       dataptrs[i] = recovi;
+       dataptrs[j] = recovj;
+
+       raid6_dual_recov(disks, PAGE_SIZE, i, j, dataptrs);
+
+       erra = memcmp(page_address(data[i]), page_address(recovi), PAGE_SIZE);
+       errb = memcmp(page_address(data[j]), page_address(recovj), PAGE_SIZE);
+
+       pr("%s(%d, %d): faila=%3d(%c)  failb=%3d(%c)  %s\n",
+          __func__, i, j, i, disk_type(i, disks), j, disk_type(j, disks),
+          (!erra && !errb) ? "OK" : !erra ? "ERRB" : !errb ? "ERRA" : "ERRAB");
+
+       dataptrs[i] = data[i];
+       dataptrs[j] = data[j];
+
+       return erra || errb;
+}
+
+static int test(int disks, int *tests)
+{
+       struct dma_async_tx_descriptor *tx;
+       struct async_submit_ctl submit;
+       struct completion cmp;
+       int err = 0;
+       int i, j;
+
+       recovi = data[disks];
+       recovj = data[disks+1];
+       spare  = data[disks+2];
+
+       makedata(disks);
+
+       /* Nuke syndromes */
+       memset(page_address(data[disks-2]), 0xee, PAGE_SIZE);
+       memset(page_address(data[disks-1]), 0xee, PAGE_SIZE);
+
+       /* Generate assumed good syndrome */
+       init_completion(&cmp);
+       init_async_submit(&submit, ASYNC_TX_ACK, NULL, callback, &cmp, addr_conv);
+       tx = async_gen_syndrome(dataptrs, 0, disks, PAGE_SIZE, &submit);
+       async_tx_issue_pending(tx);
+
+       if (wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)) == 0) {
+               pr("error: initial gen_syndrome(%d) timed out\n", disks);
+               return 1;
+       }
+
+       pr("testing the %d-disk case...\n", disks);
+       for (i = 0; i < disks-1; i++)
+               for (j = i+1; j < disks; j++) {
+                       (*tests)++;
+                       err += test_disks(i, j, disks);
+               }
+
+       return err;
+}
+
+
+static int raid6_test(void)
+{
+       int err = 0;
+       int tests = 0;
+       int i;
+
+       for (i = 0; i < NDISKS+3; i++) {
+               data[i] = alloc_page(GFP_KERNEL);
+               if (!data[i]) {
+                       while (i--)
+                               put_page(data[i]);
+                       return -ENOMEM;
+               }
+       }
+
+       /* the 4-disk and 5-disk cases are special for the recovery code */
+       if (NDISKS > 4)
+               err += test(4, &tests);
+       if (NDISKS > 5)
+               err += test(5, &tests);
+       err += test(NDISKS, &tests);
+
+       pr("\n");
+       pr("complete (%d tests, %d failure%s)\n",
+          tests, err, err == 1 ? "" : "s");
+
+       for (i = 0; i < NDISKS+3; i++)
+               put_page(data[i]);
+
+       return 0;
+}
+
+static void raid6_test_exit(void)
+{
+}
+
+/* when compiled-in wait for drivers to load first (assumes dma drivers
+ * are also compliled-in)
+ */
+late_initcall(raid6_test);
+module_exit(raid6_test_exit);
+MODULE_AUTHOR("Dan Williams <dan.j.williams@intel.com>");
+MODULE_DESCRIPTION("asynchronous RAID-6 recovery self tests");
+MODULE_LICENSE("GPL");
index 25b743a..52e6bb7 100644 (file)
@@ -28,7 +28,7 @@
 #include <linux/device.h>
 #include <linux/dca.h>
 
-#define DCA_VERSION "1.8"
+#define DCA_VERSION "1.12.1"
 
 MODULE_VERSION(DCA_VERSION);
 MODULE_LICENSE("GPL");
@@ -36,20 +36,92 @@ MODULE_AUTHOR("Intel Corporation");
 
 static DEFINE_SPINLOCK(dca_lock);
 
-static LIST_HEAD(dca_providers);
+static LIST_HEAD(dca_domains);
 
-static struct dca_provider *dca_find_provider_by_dev(struct device *dev)
+static struct pci_bus *dca_pci_rc_from_dev(struct device *dev)
 {
-       struct dca_provider *dca, *ret = NULL;
+       struct pci_dev *pdev = to_pci_dev(dev);
+       struct pci_bus *bus = pdev->bus;
 
-       list_for_each_entry(dca, &dca_providers, node) {
-               if ((!dev) || (dca->ops->dev_managed(dca, dev))) {
-                       ret = dca;
-                       break;
-               }
+       while (bus->parent)
+               bus = bus->parent;
+
+       return bus;
+}
+
+static struct dca_domain *dca_allocate_domain(struct pci_bus *rc)
+{
+       struct dca_domain *domain;
+
+       domain = kzalloc(sizeof(*domain), GFP_NOWAIT);
+       if (!domain)
+               return NULL;
+
+       INIT_LIST_HEAD(&domain->dca_providers);
+       domain->pci_rc = rc;
+
+       return domain;
+}
+
+static void dca_free_domain(struct dca_domain *domain)
+{
+       list_del(&domain->node);
+       kfree(domain);
+}
+
+static struct dca_domain *dca_find_domain(struct pci_bus *rc)
+{
+       struct dca_domain *domain;
+
+       list_for_each_entry(domain, &dca_domains, node)
+               if (domain->pci_rc == rc)
+                       return domain;
+
+       return NULL;
+}
+
+static struct dca_domain *dca_get_domain(struct device *dev)
+{
+       struct pci_bus *rc;
+       struct dca_domain *domain;
+
+       rc = dca_pci_rc_from_dev(dev);
+       domain = dca_find_domain(rc);
+
+       if (!domain) {
+               domain = dca_allocate_domain(rc);
+               if (domain)
+                       list_add(&domain->node, &dca_domains);
+       }
+
+       return domain;
+}
+
+static struct dca_provider *dca_find_provider_by_dev(struct device *dev)
+{
+       struct dca_provider *dca;
+       struct pci_bus *rc;
+       struct dca_domain *domain;
+
+       if (dev) {
+               rc = dca_pci_rc_from_dev(dev);
+               domain = dca_find_domain(rc);
+               if (!domain)
+                       return NULL;
+       } else {
+               if (!list_empty(&dca_domains))
+                       domain = list_first_entry(&dca_domains,
+                                                 struct dca_domain,
+                                                 node);
+               else
+                       return NULL;
        }
 
-       return ret;
+       list_for_each_entry(dca, &domain->dca_providers, node)
+               if ((!dev) || (dca->ops->dev_managed(dca, dev)))
+                       return dca;
+
+       return NULL;
 }
 
 /**
@@ -61,6 +133,8 @@ int dca_add_requester(struct device *dev)
        struct dca_provider *dca;
        int err, slot = -ENODEV;
        unsigned long flags;
+       struct pci_bus *pci_rc;
+       struct dca_domain *domain;
 
        if (!dev)
                return -EFAULT;
@@ -74,7 +148,14 @@ int dca_add_requester(struct device *dev)
                return -EEXIST;
        }
 
-       list_for_each_entry(dca, &dca_providers, node) {
+       pci_rc = dca_pci_rc_from_dev(dev);
+       domain = dca_find_domain(pci_rc);
+       if (!domain) {
+               spin_unlock_irqrestore(&dca_lock, flags);
+               return -ENODEV;
+       }
+
+       list_for_each_entry(dca, &domain->dca_providers, node) {
                slot = dca->ops->add_requester(dca, dev);
                if (slot >= 0)
                        break;
@@ -222,13 +303,19 @@ int register_dca_provider(struct dca_provider *dca, struct device *dev)
 {
        int err;
        unsigned long flags;
+       struct dca_domain *domain;
 
        err = dca_sysfs_add_provider(dca, dev);
        if (err)
                return err;
 
        spin_lock_irqsave(&dca_lock, flags);
-       list_add(&dca->node, &dca_providers);
+       domain = dca_get_domain(dev);
+       if (!domain) {
+               spin_unlock_irqrestore(&dca_lock, flags);
+               return -ENODEV;
+       }
+       list_add(&dca->node, &domain->dca_providers);
        spin_unlock_irqrestore(&dca_lock, flags);
 
        blocking_notifier_call_chain(&dca_provider_chain,
@@ -241,15 +328,24 @@ EXPORT_SYMBOL_GPL(register_dca_provider);
  * unregister_dca_provider - remove a dca provider
  * @dca - struct created by alloc_dca_provider()
  */
-void unregister_dca_provider(struct dca_provider *dca)
+void unregister_dca_provider(struct dca_provider *dca, struct device *dev)
 {
        unsigned long flags;
+       struct pci_bus *pci_rc;
+       struct dca_domain *domain;
 
        blocking_notifier_call_chain(&dca_provider_chain,
                                     DCA_PROVIDER_REMOVE, NULL);
 
        spin_lock_irqsave(&dca_lock, flags);
+
        list_del(&dca->node);
+
+       pci_rc = dca_pci_rc_from_dev(dev);
+       domain = dca_find_domain(pci_rc);
+       if (list_empty(&domain->dca_providers))
+               dca_free_domain(domain);
+
        spin_unlock_irqrestore(&dca_lock, flags);
 
        dca_sysfs_remove_provider(dca);
@@ -276,7 +372,7 @@ EXPORT_SYMBOL_GPL(dca_unregister_notify);
 
 static int __init dca_init(void)
 {
-       printk(KERN_ERR "dca service started, version %s\n", DCA_VERSION);
+       pr_info("dca service started, version %s\n", DCA_VERSION);
        return dca_sysfs_init();
 }
 
index 81e1020..5903a88 100644 (file)
@@ -17,11 +17,15 @@ if DMADEVICES
 
 comment "DMA Devices"
 
+config ASYNC_TX_DISABLE_CHANNEL_SWITCH
+       bool
+
 config INTEL_IOATDMA
        tristate "Intel I/OAT DMA support"
        depends on PCI && X86
        select DMA_ENGINE
        select DCA
+       select ASYNC_TX_DISABLE_CHANNEL_SWITCH
        help
          Enable support for the Intel(R) I/OAT DMA engine present
          in recent Intel Xeon chipsets.
@@ -97,6 +101,14 @@ config TXX9_DMAC
          Support the TXx9 SoC internal DMA controller.  This can be
          integrated in chips such as the Toshiba TX4927/38/39.
 
+config SH_DMAE
+       tristate "Renesas SuperH DMAC support"
+       depends on SUPERH && SH_DMA
+       depends on !SH_DMA_API
+       select DMA_ENGINE
+       help
+         Enable support for the Renesas SuperH DMA controllers.
+
 config DMA_ENGINE
        bool
 
@@ -116,7 +128,7 @@ config NET_DMA
 
 config ASYNC_TX_DMA
        bool "Async_tx: Offload support for the async_tx api"
-       depends on DMA_ENGINE && !HIGHMEM64G
+       depends on DMA_ENGINE
        help
          This allows the async_tx api to take advantage of offload engines for
          memcpy, memset, xor, and raid6 p+q operations.  If your platform has
index 40e1e00..eca71ba 100644 (file)
@@ -1,8 +1,7 @@
 obj-$(CONFIG_DMA_ENGINE) += dmaengine.o
 obj-$(CONFIG_NET_DMA) += iovlock.o
 obj-$(CONFIG_DMATEST) += dmatest.o
-obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o
-ioatdma-objs := ioat.o ioat_dma.o ioat_dca.o
+obj-$(CONFIG_INTEL_IOATDMA) += ioat/
 obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.o
 obj-$(CONFIG_FSL_DMA) += fsldma.o
 obj-$(CONFIG_MV_XOR) += mv_xor.o
@@ -10,3 +9,4 @@ obj-$(CONFIG_DW_DMAC) += dw_dmac.o
 obj-$(CONFIG_AT_HDMAC) += at_hdmac.o
 obj-$(CONFIG_MX3_IPU) += ipu/
 obj-$(CONFIG_TXX9_DMAC) += txx9dmac.o
+obj-$(CONFIG_SH_DMAE) += shdma.o
index c8522e6..7585c41 100644 (file)
@@ -87,6 +87,7 @@ static struct at_desc *atc_alloc_descriptor(struct dma_chan *chan,
        desc = dma_pool_alloc(atdma->dma_desc_pool, gfp_flags, &phys);
        if (desc) {
                memset(desc, 0, sizeof(struct at_desc));
+               INIT_LIST_HEAD(&desc->tx_list);
                dma_async_tx_descriptor_init(&desc->txd, chan);
                /* txd.flags will be overwritten in prep functions */
                desc->txd.flags = DMA_CTRL_ACK;
@@ -150,11 +151,11 @@ static void atc_desc_put(struct at_dma_chan *atchan, struct at_desc *desc)
                struct at_desc *child;
 
                spin_lock_bh(&atchan->lock);
-               list_for_each_entry(child, &desc->txd.tx_list, desc_node)
+               list_for_each_entry(child, &desc->tx_list, desc_node)
                        dev_vdbg(chan2dev(&atchan->chan_common),
                                        "moving child desc %p to freelist\n",
                                        child);
-               list_splice_init(&desc->txd.tx_list, &atchan->free_list);
+               list_splice_init(&desc->tx_list, &atchan->free_list);
                dev_vdbg(chan2dev(&atchan->chan_common),
                         "moving desc %p to freelist\n", desc);
                list_add(&desc->desc_node, &atchan->free_list);
@@ -247,30 +248,33 @@ atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc)
        param = txd->callback_param;
 
        /* move children to free_list */
-       list_splice_init(&txd->tx_list, &atchan->free_list);
+       list_splice_init(&desc->tx_list, &atchan->free_list);
        /* move myself to free_list */
        list_move(&desc->desc_node, &atchan->free_list);
 
        /* unmap dma addresses */
-       if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
-               if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
-                       dma_unmap_single(chan2parent(&atchan->chan_common),
-                                       desc->lli.daddr,
-                                       desc->len, DMA_FROM_DEVICE);
-               else
-                       dma_unmap_page(chan2parent(&atchan->chan_common),
-                                       desc->lli.daddr,
-                                       desc->len, DMA_FROM_DEVICE);
-       }
-       if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
-               if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
-                       dma_unmap_single(chan2parent(&atchan->chan_common),
-                                       desc->lli.saddr,
-                                       desc->len, DMA_TO_DEVICE);
-               else
-                       dma_unmap_page(chan2parent(&atchan->chan_common),
-                                       desc->lli.saddr,
-                                       desc->len, DMA_TO_DEVICE);
+       if (!atchan->chan_common.private) {
+               struct device *parent = chan2parent(&atchan->chan_common);
+               if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
+                       if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
+                               dma_unmap_single(parent,
+                                               desc->lli.daddr,
+                                               desc->len, DMA_FROM_DEVICE);
+                       else
+                               dma_unmap_page(parent,
+                                               desc->lli.daddr,
+                                               desc->len, DMA_FROM_DEVICE);
+               }
+               if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
+                       if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
+                               dma_unmap_single(parent,
+                                               desc->lli.saddr,
+                                               desc->len, DMA_TO_DEVICE);
+                       else
+                               dma_unmap_page(parent,
+                                               desc->lli.saddr,
+                                               desc->len, DMA_TO_DEVICE);
+               }
        }
 
        /*
@@ -334,7 +338,7 @@ static void atc_cleanup_descriptors(struct at_dma_chan *atchan)
                        /* This one is currently in progress */
                        return;
 
-               list_for_each_entry(child, &desc->txd.tx_list, desc_node)
+               list_for_each_entry(child, &desc->tx_list, desc_node)
                        if (!(child->lli.ctrla & ATC_DONE))
                                /* Currently in progress */
                                return;
@@ -407,7 +411,7 @@ static void atc_handle_error(struct at_dma_chan *atchan)
        dev_crit(chan2dev(&atchan->chan_common),
                        "  cookie: %d\n", bad_desc->txd.cookie);
        atc_dump_lli(atchan, &bad_desc->lli);
-       list_for_each_entry(child, &bad_desc->txd.tx_list, desc_node)
+       list_for_each_entry(child, &bad_desc->tx_list, desc_node)
                atc_dump_lli(atchan, &child->lli);
 
        /* Pretend the descriptor completed successfully */
@@ -587,7 +591,7 @@ atc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
                        prev->lli.dscr = desc->txd.phys;
                        /* insert the link descriptor to the LD ring */
                        list_add_tail(&desc->desc_node,
-                                       &first->txd.tx_list);
+                                       &first->tx_list);
                }
                prev = desc;
        }
@@ -646,8 +650,6 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 
        reg_width = atslave->reg_width;
 
-       sg_len = dma_map_sg(chan2parent(chan), sgl, sg_len, direction);
-
        ctrla = ATC_DEFAULT_CTRLA | atslave->ctrla;
        ctrlb = ATC_DEFAULT_CTRLB | ATC_IEN;
 
@@ -687,7 +689,7 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
                                prev->lli.dscr = desc->txd.phys;
                                /* insert the link descriptor to the LD ring */
                                list_add_tail(&desc->desc_node,
-                                               &first->txd.tx_list);
+                                               &first->tx_list);
                        }
                        prev = desc;
                        total_len += len;
@@ -729,7 +731,7 @@ atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
                                prev->lli.dscr = desc->txd.phys;
                                /* insert the link descriptor to the LD ring */
                                list_add_tail(&desc->desc_node,
-                                               &first->txd.tx_list);
+                                               &first->tx_list);
                        }
                        prev = desc;
                        total_len += len;
index 4c972af..495457e 100644 (file)
@@ -165,6 +165,7 @@ struct at_desc {
        struct at_lli                   lli;
 
        /* THEN values for driver housekeeping */
+       struct list_head                tx_list;
        struct dma_async_tx_descriptor  txd;
        struct list_head                desc_node;
        size_t                          len;
index 5a87384..bd0b248 100644 (file)
@@ -608,6 +608,40 @@ void dmaengine_put(void)
 }
 EXPORT_SYMBOL(dmaengine_put);
 
+static bool device_has_all_tx_types(struct dma_device *device)
+{
+       /* A device that satisfies this test has channels that will never cause
+        * an async_tx channel switch event as all possible operation types can
+        * be handled.
+        */
+       #ifdef CONFIG_ASYNC_TX_DMA
+       if (!dma_has_cap(DMA_INTERRUPT, device->cap_mask))
+               return false;
+       #endif
+
+       #if defined(CONFIG_ASYNC_MEMCPY) || defined(CONFIG_ASYNC_MEMCPY_MODULE)
+       if (!dma_has_cap(DMA_MEMCPY, device->cap_mask))
+               return false;
+       #endif
+
+       #if defined(CONFIG_ASYNC_MEMSET) || defined(CONFIG_ASYNC_MEMSET_MODULE)
+       if (!dma_has_cap(DMA_MEMSET, device->cap_mask))
+               return false;
+       #endif
+
+       #if defined(CONFIG_ASYNC_XOR) || defined(CONFIG_ASYNC_XOR_MODULE)
+       if (!dma_has_cap(DMA_XOR, device->cap_mask))
+               return false;
+       #endif
+
+       #if defined(CONFIG_ASYNC_PQ) || defined(CONFIG_ASYNC_PQ_MODULE)
+       if (!dma_has_cap(DMA_PQ, device->cap_mask))
+               return false;
+       #endif
+
+       return true;
+}
+
 static int get_dma_id(struct dma_device *device)
 {
        int rc;
@@ -644,8 +678,12 @@ int dma_async_device_register(struct dma_device *device)
                !device->device_prep_dma_memcpy);
        BUG_ON(dma_has_cap(DMA_XOR, device->cap_mask) &&
                !device->device_prep_dma_xor);
-       BUG_ON(dma_has_cap(DMA_ZERO_SUM, device->cap_mask) &&
-               !device->device_prep_dma_zero_sum);
+       BUG_ON(dma_has_cap(DMA_XOR_VAL, device->cap_mask) &&
+               !device->device_prep_dma_xor_val);
+       BUG_ON(dma_has_cap(DMA_PQ, device->cap_mask) &&
+               !device->device_prep_dma_pq);
+       BUG_ON(dma_has_cap(DMA_PQ_VAL, device->cap_mask) &&
+               !device->device_prep_dma_pq_val);
        BUG_ON(dma_has_cap(DMA_MEMSET, device->cap_mask) &&
                !device->device_prep_dma_memset);
        BUG_ON(dma_has_cap(DMA_INTERRUPT, device->cap_mask) &&
@@ -661,6 +699,12 @@ int dma_async_device_register(struct dma_device *device)
        BUG_ON(!device->device_issue_pending);
        BUG_ON(!device->dev);
 
+       /* note: this only matters in the
+        * CONFIG_ASYNC_TX_DISABLE_CHANNEL_SWITCH=y case
+        */
+       if (device_has_all_tx_types(device))
+               dma_cap_set(DMA_ASYNC_TX, device->cap_mask);
+
        idr_ref = kmalloc(sizeof(*idr_ref), GFP_KERNEL);
        if (!idr_ref)
                return -ENOMEM;
@@ -933,55 +977,29 @@ void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,
 {
        tx->chan = chan;
        spin_lock_init(&tx->lock);
-       INIT_LIST_HEAD(&tx->tx_list);
 }
 EXPORT_SYMBOL(dma_async_tx_descriptor_init);
 
 /* dma_wait_for_async_tx - spin wait for a transaction to complete
  * @tx: in-flight transaction to wait on
- *
- * This routine assumes that tx was obtained from a call to async_memcpy,
- * async_xor, async_memset, etc which ensures that tx is "in-flight" (prepped
- * and submitted).  Walking the parent chain is only meant to cover for DMA
- * drivers that do not implement the DMA_INTERRUPT capability and may race with
- * the driver's descriptor cleanup routine.
  */
 enum dma_status
 dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx)
 {
-       enum dma_status status;
-       struct dma_async_tx_descriptor *iter;
-       struct dma_async_tx_descriptor *parent;
+       unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000);
 
        if (!tx)
                return DMA_SUCCESS;
 
-       WARN_ONCE(tx->parent, "%s: speculatively walking dependency chain for"
-                 " %s\n", __func__, dma_chan_name(tx->chan));
-
-       /* poll through the dependency chain, return when tx is complete */
-       do {
-               iter = tx;
-
-               /* find the root of the unsubmitted dependency chain */
-               do {
-                       parent = iter->parent;
-                       if (!parent)
-                               break;
-                       else
-                               iter = parent;
-               } while (parent);
-
-               /* there is a small window for ->parent == NULL and
-                * ->cookie == -EBUSY
-                */
-               while (iter->cookie == -EBUSY)
-                       cpu_relax();
-
-               status = dma_sync_wait(iter->chan, iter->cookie);
-       } while (status == DMA_IN_PROGRESS || (iter != tx));
-
-       return status;
+       while (tx->cookie == -EBUSY) {
+               if (time_after_eq(jiffies, dma_sync_wait_timeout)) {
+                       pr_err("%s timeout waiting for descriptor submission\n",
+                               __func__);
+                       return DMA_ERROR;
+               }
+               cpu_relax();
+       }
+       return dma_sync_wait(tx->chan, tx->cookie);
 }
 EXPORT_SYMBOL_GPL(dma_wait_for_async_tx);
 
index d93017f..a32a4cf 100644 (file)
@@ -48,6 +48,11 @@ module_param(xor_sources, uint, S_IRUGO);
 MODULE_PARM_DESC(xor_sources,
                "Number of xor source buffers (default: 3)");
 
+static unsigned int pq_sources = 3;
+module_param(pq_sources, uint, S_IRUGO);
+MODULE_PARM_DESC(pq_sources,
+               "Number of p+q source buffers (default: 3)");
+
 /*
  * Initialization patterns. All bytes in the source buffer has bit 7
  * set, all bytes in the destination buffer has bit 7 cleared.
@@ -232,6 +237,7 @@ static int dmatest_func(void *data)
        dma_cookie_t            cookie;
        enum dma_status         status;
        enum dma_ctrl_flags     flags;
+       u8                      pq_coefs[pq_sources];
        int                     ret;
        int                     src_cnt;
        int                     dst_cnt;
@@ -248,6 +254,11 @@ static int dmatest_func(void *data)
        else if (thread->type == DMA_XOR) {
                src_cnt = xor_sources | 1; /* force odd to ensure dst = src */
                dst_cnt = 1;
+       } else if (thread->type == DMA_PQ) {
+               src_cnt = pq_sources | 1; /* force odd to ensure dst = src */
+               dst_cnt = 2;
+               for (i = 0; i < pq_sources; i++)
+                       pq_coefs[i] = 1;
        } else
                goto err_srcs;
 
@@ -283,6 +294,7 @@ static int dmatest_func(void *data)
                dma_addr_t dma_dsts[dst_cnt];
                struct completion cmp;
                unsigned long tmo = msecs_to_jiffies(3000);
+               u8 align = 0;
 
                total_tests++;
 
@@ -290,6 +302,18 @@ static int dmatest_func(void *data)
                src_off = dmatest_random() % (test_buf_size - len + 1);
                dst_off = dmatest_random() % (test_buf_size - len + 1);
 
+               /* honor alignment restrictions */
+               if (thread->type == DMA_MEMCPY)
+                       align = dev->copy_align;
+               else if (thread->type == DMA_XOR)
+                       align = dev->xor_align;
+               else if (thread->type == DMA_PQ)
+                       align = dev->pq_align;
+
+               len = (len >> align) << align;
+               src_off = (src_off >> align) << align;
+               dst_off = (dst_off >> align) << align;
+
                dmatest_init_srcs(thread->srcs, src_off, len);
                dmatest_init_dsts(thread->dsts, dst_off, len);
 
@@ -306,6 +330,7 @@ static int dmatest_func(void *data)
                                                     DMA_BIDIRECTIONAL);
                }
 
+
                if (thread->type == DMA_MEMCPY)
                        tx = dev->device_prep_dma_memcpy(chan,
                                                         dma_dsts[0] + dst_off,
@@ -316,6 +341,15 @@ static int dmatest_func(void *data)
                                                      dma_dsts[0] + dst_off,
                                                      dma_srcs, xor_sources,
                                                      len, flags);
+               else if (thread->type == DMA_PQ) {
+                       dma_addr_t dma_pq[dst_cnt];
+
+                       for (i = 0; i < dst_cnt; i++)
+                               dma_pq[i] = dma_dsts[i] + dst_off;
+                       tx = dev->device_prep_dma_pq(chan, dma_pq, dma_srcs,
+                                                    pq_sources, pq_coefs,
+                                                    len, flags);
+               }
 
                if (!tx) {
                        for (i = 0; i < src_cnt; i++)
@@ -459,6 +493,8 @@ static int dmatest_add_threads(struct dmatest_chan *dtc, enum dma_transaction_ty
                op = "copy";
        else if (type == DMA_XOR)
                op = "xor";
+       else if (type == DMA_PQ)
+               op = "pq";
        else
                return -EINVAL;
 
@@ -514,6 +550,10 @@ static int dmatest_add_channel(struct dma_chan *chan)
                cnt = dmatest_add_threads(dtc, DMA_XOR);
                thread_count += cnt > 0 ? cnt : 0;
        }
+       if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) {
+               cnt = dmatest_add_threads(dtc, DMA_PQ);
+               thread_count += cnt > 0 ?: 0;
+       }
 
        pr_info("dmatest: Started %u threads using %s\n",
                thread_count, dma_chan_name(chan));
index 933c143..2eea823 100644 (file)
@@ -116,7 +116,7 @@ static void dwc_sync_desc_for_cpu(struct dw_dma_chan *dwc, struct dw_desc *desc)
 {
        struct dw_desc  *child;
 
-       list_for_each_entry(child, &desc->txd.tx_list, desc_node)
+       list_for_each_entry(child, &desc->tx_list, desc_node)
                dma_sync_single_for_cpu(chan2parent(&dwc->chan),
                                child->txd.phys, sizeof(child->lli),
                                DMA_TO_DEVICE);
@@ -137,11 +137,11 @@ static void dwc_desc_put(struct dw_dma_chan *dwc, struct dw_desc *desc)
                dwc_sync_desc_for_cpu(dwc, desc);
 
                spin_lock_bh(&dwc->lock);
-               list_for_each_entry(child, &desc->txd.tx_list, desc_node)
+               list_for_each_entry(child, &desc->tx_list, desc_node)
                        dev_vdbg(chan2dev(&dwc->chan),
                                        "moving child desc %p to freelist\n",
                                        child);
-               list_splice_init(&desc->txd.tx_list, &dwc->free_list);
+               list_splice_init(&desc->tx_list, &dwc->free_list);
                dev_vdbg(chan2dev(&dwc->chan), "moving desc %p to freelist\n", desc);
                list_add(&desc->desc_node, &dwc->free_list);
                spin_unlock_bh(&dwc->lock);
@@ -209,19 +209,28 @@ dwc_descriptor_complete(struct dw_dma_chan *dwc, struct dw_desc *desc)
        param = txd->callback_param;
 
        dwc_sync_desc_for_cpu(dwc, desc);
-       list_splice_init(&txd->tx_list, &dwc->free_list);
+       list_splice_init(&desc->tx_list, &dwc->free_list);
        list_move(&desc->desc_node, &dwc->free_list);
 
-       /*
-        * We use dma_unmap_page() regardless of how the buffers were
-        * mapped before they were submitted...
-        */
-       if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP))
-               dma_unmap_page(chan2parent(&dwc->chan), desc->lli.dar,
-                              desc->len, DMA_FROM_DEVICE);
-       if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP))
-               dma_unmap_page(chan2parent(&dwc->chan), desc->lli.sar,
-                              desc->len, DMA_TO_DEVICE);
+       if (!dwc->chan.private) {
+               struct device *parent = chan2parent(&dwc->chan);
+               if (!(txd->flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
+                       if (txd->flags & DMA_COMPL_DEST_UNMAP_SINGLE)
+                               dma_unmap_single(parent, desc->lli.dar,
+                                               desc->len, DMA_FROM_DEVICE);
+                       else
+                               dma_unmap_page(parent, desc->lli.dar,
+                                               desc->len, DMA_FROM_DEVICE);
+               }
+               if (!(txd->flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
+                       if (txd->flags & DMA_COMPL_SRC_UNMAP_SINGLE)
+                               dma_unmap_single(parent, desc->lli.sar,
+                                               desc->len, DMA_TO_DEVICE);
+                       else
+                               dma_unmap_page(parent, desc->lli.sar,
+                                               desc->len, DMA_TO_DEVICE);
+               }
+       }
 
        /*
         * The API requires that no submissions are done from a
@@ -289,7 +298,7 @@ static void dwc_scan_descriptors(struct dw_dma *dw, struct dw_dma_chan *dwc)
                        /* This one is currently in progress */
                        return;
 
-               list_for_each_entry(child, &desc->txd.tx_list, desc_node)
+               list_for_each_entry(child, &desc->tx_list, desc_node)
                        if (child->lli.llp == llp)
                                /* Currently in progress */
                                return;
@@ -356,7 +365,7 @@ static void dwc_handle_error(struct dw_dma *dw, struct dw_dma_chan *dwc)
        dev_printk(KERN_CRIT, chan2dev(&dwc->chan),
                        "  cookie: %d\n", bad_desc->txd.cookie);
        dwc_dump_lli(dwc, &bad_desc->lli);
-       list_for_each_entry(child, &bad_desc->txd.tx_list, desc_node)
+       list_for_each_entry(child, &bad_desc->tx_list, desc_node)
                dwc_dump_lli(dwc, &child->lli);
 
        /* Pretend the descriptor completed successfully */
@@ -608,7 +617,7 @@ dwc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
                                        prev->txd.phys, sizeof(prev->lli),
                                        DMA_TO_DEVICE);
                        list_add_tail(&desc->desc_node,
-                                       &first->txd.tx_list);
+                                       &first->tx_list);
                }
                prev = desc;
        }
@@ -658,8 +667,6 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
        reg_width = dws->reg_width;
        prev = first = NULL;
 
-       sg_len = dma_map_sg(chan2parent(chan), sgl, sg_len, direction);
-
        switch (direction) {
        case DMA_TO_DEVICE:
                ctllo = (DWC_DEFAULT_CTLLO
@@ -700,7 +707,7 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
                                                sizeof(prev->lli),
                                                DMA_TO_DEVICE);
                                list_add_tail(&desc->desc_node,
-                                               &first->txd.tx_list);
+                                               &first->tx_list);
                        }
                        prev = desc;
                        total_len += len;
@@ -746,7 +753,7 @@ dwc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
                                                sizeof(prev->lli),
                                                DMA_TO_DEVICE);
                                list_add_tail(&desc->desc_node,
-                                               &first->txd.tx_list);
+                                               &first->tx_list);
                        }
                        prev = desc;
                        total_len += len;
@@ -902,6 +909,7 @@ static int dwc_alloc_chan_resources(struct dma_chan *chan)
                        break;
                }
 
+               INIT_LIST_HEAD(&desc->tx_list);
                dma_async_tx_descriptor_init(&desc->txd, chan);
                desc->txd.tx_submit = dwc_tx_submit;
                desc->txd.flags = DMA_CTRL_ACK;
index 13a5807..d9a939f 100644 (file)
@@ -217,6 +217,7 @@ struct dw_desc {
 
        /* THEN values for driver housekeeping */
        struct list_head                desc_node;
+       struct list_head                tx_list;
        struct dma_async_tx_descriptor  txd;
        size_t                          len;
 };
index ef87a89..296f9e7 100644 (file)
@@ -34,6 +34,7 @@
 #include <linux/dmapool.h>
 #include <linux/of_platform.h>
 
+#include <asm/fsldma.h>
 #include "fsldma.h"
 
 static void dma_init(struct fsl_dma_chan *fsl_chan)
@@ -280,28 +281,40 @@ static void fsl_chan_set_dest_loop_size(struct fsl_dma_chan *fsl_chan, int size)
 }
 
 /**
- * fsl_chan_toggle_ext_pause - Toggle channel external pause status
+ * fsl_chan_set_request_count - Set DMA Request Count for external control
  * @fsl_chan : Freescale DMA channel
- * @size     : Pause control size, 0 for disable external pause control.
- *             The maximum is 1024.
+ * @size     : Number of bytes to transfer in a single request
+ *
+ * The Freescale DMA channel can be controlled by the external signal DREQ#.
+ * The DMA request count is how many bytes are allowed to transfer before
+ * pausing the channel, after which a new assertion of DREQ# resumes channel
+ * operation.
  *
- * The Freescale DMA channel can be controlled by the external
- * signal DREQ#. The pause control size is how many bytes are allowed
- * to transfer before pausing the channel, after which a new assertion
- * of DREQ# resumes channel operation.
+ * A size of 0 disables external pause control. The maximum size is 1024.
  */
-static void fsl_chan_toggle_ext_pause(struct fsl_dma_chan *fsl_chan, int size)
+static void fsl_chan_set_request_count(struct fsl_dma_chan *fsl_chan, int size)
 {
-       if (size > 1024)
-               return;
+       BUG_ON(size > 1024);
+       DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr,
+               DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32)
+                       | ((__ilog2(size) << 24) & 0x0f000000),
+               32);
+}
 
-       if (size) {
-               DMA_OUT(fsl_chan, &fsl_chan->reg_base->mr,
-                       DMA_IN(fsl_chan, &fsl_chan->reg_base->mr, 32)
-                               | ((__ilog2(size) << 24) & 0x0f000000),
-                       32);
+/**
+ * fsl_chan_toggle_ext_pause - Toggle channel external pause status
+ * @fsl_chan : Freescale DMA channel
+ * @enable   : 0 is disabled, 1 is enabled.
+ *
+ * The Freescale DMA channel can be controlled by the external signal DREQ#.
+ * The DMA Request Count feature should be used in addition to this feature
+ * to set the number of bytes to transfer before pausing the channel.
+ */
+static void fsl_chan_toggle_ext_pause(struct fsl_dma_chan *fsl_chan, int enable)
+{
+       if (enable)
                fsl_chan->feature |= FSL_DMA_CHAN_PAUSE_EXT;
-       } else
+       else
                fsl_chan->feature &= ~FSL_DMA_CHAN_PAUSE_EXT;
 }
 
@@ -326,7 +339,8 @@ static void fsl_chan_toggle_ext_start(struct fsl_dma_chan *fsl_chan, int enable)
 static dma_cookie_t fsl_dma_tx_submit(struct dma_async_tx_descriptor *tx)
 {
        struct fsl_dma_chan *fsl_chan = to_fsl_chan(tx->chan);
-       struct fsl_desc_sw *desc;
+       struct fsl_desc_sw *desc = tx_to_fsl_desc(tx);
+       struct fsl_desc_sw *child;
        unsigned long flags;
        dma_cookie_t cookie;
 
@@ -334,7 +348,7 @@ static dma_cookie_t fsl_dma_tx_submit(struct dma_async_tx_descriptor *tx)
        spin_lock_irqsave(&fsl_chan->desc_lock, flags);
 
        cookie = fsl_chan->common.cookie;
-       list_for_each_entry(desc, &tx->tx_list, node) {
+       list_for_each_entry(child, &desc->tx_list, node) {
                cookie++;
                if (cookie < 0)
                        cookie = 1;
@@ -343,8 +357,8 @@ static dma_cookie_t fsl_dma_tx_submit(struct dma_async_tx_descriptor *tx)
        }
 
        fsl_chan->common.cookie = cookie;
-       append_ld_queue(fsl_chan, tx_to_fsl_desc(tx));
-       list_splice_init(&tx->tx_list, fsl_chan->ld_queue.prev);
+       append_ld_queue(fsl_chan, desc);
+       list_splice_init(&desc->tx_list, fsl_chan->ld_queue.prev);
 
        spin_unlock_irqrestore(&fsl_chan->desc_lock, flags);
 
@@ -366,6 +380,7 @@ static struct fsl_desc_sw *fsl_dma_alloc_descriptor(
        desc_sw = dma_pool_alloc(fsl_chan->desc_pool, GFP_ATOMIC, &pdesc);
        if (desc_sw) {
                memset(desc_sw, 0, sizeof(struct fsl_desc_sw));
+               INIT_LIST_HEAD(&desc_sw->tx_list);
                dma_async_tx_descriptor_init(&desc_sw->async_tx,
                                                &fsl_chan->common);
                desc_sw->async_tx.tx_submit = fsl_dma_tx_submit;
@@ -455,7 +470,7 @@ fsl_dma_prep_interrupt(struct dma_chan *chan, unsigned long flags)
        new->async_tx.flags = flags;
 
        /* Insert the link descriptor to the LD ring */
-       list_add_tail(&new->node, &new->async_tx.tx_list);
+       list_add_tail(&new->node, &new->tx_list);
 
        /* Set End-of-link to the last link descriptor of new list*/
        set_ld_eol(fsl_chan, new);
@@ -513,7 +528,7 @@ static struct dma_async_tx_descriptor *fsl_dma_prep_memcpy(
                dma_dest += copy;
 
                /* Insert the link descriptor to the LD ring */
-               list_add_tail(&new->node, &first->async_tx.tx_list);
+               list_add_tail(&new->node, &first->tx_list);
        } while (len);
 
        new->async_tx.flags = flags; /* client is in control of this ack */
@@ -528,7 +543,7 @@ fail:
        if (!first)
                return NULL;
 
-       list = &first->async_tx.tx_list;
+       list = &first->tx_list;
        list_for_each_entry_safe_reverse(new, prev, list, node) {
                list_del(&new->node);
                dma_pool_free(fsl_chan->desc_pool, new, new->async_tx.phys);
@@ -538,6 +553,229 @@ fail:
 }
 
 /**
+ * fsl_dma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction
+ * @chan: DMA channel
+ * @sgl: scatterlist to transfer to/from
+ * @sg_len: number of entries in @scatterlist
+ * @direction: DMA direction
+ * @flags: DMAEngine flags
+ *
+ * Prepare a set of descriptors for a DMA_SLAVE transaction. Following the
+ * DMA_SLAVE API, this gets the device-specific information from the
+ * chan->private variable.
+ */
+static struct dma_async_tx_descriptor *fsl_dma_prep_slave_sg(
+       struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len,
+       enum dma_data_direction direction, unsigned long flags)
+{
+       struct fsl_dma_chan *fsl_chan;
+       struct fsl_desc_sw *first = NULL, *prev = NULL, *new = NULL;
+       struct fsl_dma_slave *slave;
+       struct list_head *tx_list;
+       size_t copy;
+
+       int i;
+       struct scatterlist *sg;
+       size_t sg_used;
+       size_t hw_used;
+       struct fsl_dma_hw_addr *hw;
+       dma_addr_t dma_dst, dma_src;
+
+       if (!chan)
+               return NULL;
+
+       if (!chan->private)
+               return NULL;
+
+       fsl_chan = to_fsl_chan(chan);
+       slave = chan->private;
+
+       if (list_empty(&slave->addresses))
+               return NULL;
+
+       hw = list_first_entry(&slave->addresses, struct fsl_dma_hw_addr, entry);
+       hw_used = 0;
+
+       /*
+        * Build the hardware transaction to copy from the scatterlist to
+        * the hardware, or from the hardware to the scatterlist
+        *
+        * If you are copying from the hardware to the scatterlist and it
+        * takes two hardware entries to fill an entire page, then both
+        * hardware entries will be coalesced into the same page
+        *
+        * If you are copying from the scatterlist to the hardware and a
+        * single page can fill two hardware entries, then the data will
+        * be read out of the page into the first hardware entry, and so on
+        */
+       for_each_sg(sgl, sg, sg_len, i) {
+               sg_used = 0;
+
+               /* Loop until the entire scatterlist entry is used */
+               while (sg_used < sg_dma_len(sg)) {
+
+                       /*
+                        * If we've used up the current hardware address/length
+                        * pair, we need to load a new one
+                        *
+                        * This is done in a while loop so that descriptors with
+                        * length == 0 will be skipped
+                        */
+                       while (hw_used >= hw->length) {
+
+                               /*
+                                * If the current hardware entry is the last
+                                * entry in the list, we're finished
+                                */
+                               if (list_is_last(&hw->entry, &slave->addresses))
+                                       goto finished;
+
+                               /* Get the next hardware address/length pair */
+                               hw = list_entry(hw->entry.next,
+                                               struct fsl_dma_hw_addr, entry);
+                               hw_used = 0;
+                       }
+
+                       /* Allocate the link descriptor from DMA pool */
+                       new = fsl_dma_alloc_descriptor(fsl_chan);
+                       if (!new) {
+                               dev_err(fsl_chan->dev, "No free memory for "
+                                                      "link descriptor\n");
+                               goto fail;
+                       }
+#ifdef FSL_DMA_LD_DEBUG
+                       dev_dbg(fsl_chan->dev, "new link desc alloc %p\n", new);
+#endif
+
+                       /*
+                        * Calculate the maximum number of bytes to transfer,
+                        * making sure it is less than the DMA controller limit
+                        */
+                       copy = min_t(size_t, sg_dma_len(sg) - sg_used,
+                                            hw->length - hw_used);
+                       copy = min_t(size_t, copy, FSL_DMA_BCR_MAX_CNT);
+
+                       /*
+                        * DMA_FROM_DEVICE
+                        * from the hardware to the scatterlist
+                        *
+                        * DMA_TO_DEVICE
+                        * from the scatterlist to the hardware
+                        */
+                       if (direction == DMA_FROM_DEVICE) {
+                               dma_src = hw->address + hw_used;
+                               dma_dst = sg_dma_address(sg) + sg_used;
+                       } else {
+                               dma_src = sg_dma_address(sg) + sg_used;
+                               dma_dst = hw->address + hw_used;
+                       }
+
+                       /* Fill in the descriptor */
+                       set_desc_cnt(fsl_chan, &new->hw, copy);
+                       set_desc_src(fsl_chan, &new->hw, dma_src);
+                       set_desc_dest(fsl_chan, &new->hw, dma_dst);
+
+                       /*
+                        * If this is not the first descriptor, chain the
+                        * current descriptor after the previous descriptor
+                        */
+                       if (!first) {
+                               first = new;
+                       } else {
+                               set_desc_next(fsl_chan, &prev->hw,
+                                             new->async_tx.phys);
+                       }
+
+                       new->async_tx.cookie = 0;
+                       async_tx_ack(&new->async_tx);
+
+                       prev = new;
+                       sg_used += copy;
+                       hw_used += copy;
+
+                       /* Insert the link descriptor into the LD ring */
+                       list_add_tail(&new->node, &first->tx_list);
+               }
+       }
+
+finished:
+
+       /* All of the hardware address/length pairs had length == 0 */
+       if (!first || !new)
+               return NULL;
+
+       new->async_tx.flags = flags;
+       new->async_tx.cookie = -EBUSY;
+
+       /* Set End-of-link to the last link descriptor of new list */
+       set_ld_eol(fsl_chan, new);
+
+       /* Enable extra controller features */
+       if (fsl_chan->set_src_loop_size)
+               fsl_chan->set_src_loop_size(fsl_chan, slave->src_loop_size);
+
+       if (fsl_chan->set_dest_loop_size)
+               fsl_chan->set_dest_loop_size(fsl_chan, slave->dst_loop_size);
+
+       if (fsl_chan->toggle_ext_start)
+               fsl_chan->toggle_ext_start(fsl_chan, slave->external_start);
+
+       if (fsl_chan->toggle_ext_pause)
+               fsl_chan->toggle_ext_pause(fsl_chan, slave->external_pause);
+
+       if (fsl_chan->set_request_count)
+               fsl_chan->set_request_count(fsl_chan, slave->request_count);
+
+       return &first->async_tx;
+
+fail:
+       /* If first was not set, then we failed to allocate the very first
+        * descriptor, and we're done */
+       if (!first)
+               return NULL;
+
+       /*
+        * First is set, so all of the descriptors we allocated have been added
+        * to first->tx_list, INCLUDING "first" itself. Therefore we
+        * must traverse the list backwards freeing each descriptor in turn
+        *
+        * We're re-using variables for the loop, oh well
+        */
+       tx_list = &first->tx_list;
+       list_for_each_entry_safe_reverse(new, prev, tx_list, node) {
+               list_del_init(&new->node);
+               dma_pool_free(fsl_chan->desc_pool, new, new->async_tx.phys);
+       }
+
+       return NULL;
+}
+
+static void fsl_dma_device_terminate_all(struct dma_chan *chan)
+{
+       struct fsl_dma_chan *fsl_chan;
+       struct fsl_desc_sw *desc, *tmp;
+       unsigned long flags;
+
+       if (!chan)
+               return;
+
+       fsl_chan = to_fsl_chan(chan);
+
+       /* Halt the DMA engine */
+       dma_halt(fsl_chan);
+
+       spin_lock_irqsave(&fsl_chan->desc_lock, flags);
+
+       /* Remove and free all of the descriptors in the LD queue */
+       list_for_each_entry_safe(desc, tmp, &fsl_chan->ld_queue, node) {
+               list_del(&desc->node);
+               dma_pool_free(fsl_chan->desc_pool, desc, desc->async_tx.phys);
+       }
+
+       spin_unlock_irqrestore(&fsl_chan->desc_lock, flags);
+}
+
+/**
  * fsl_dma_update_completed_cookie - Update the completed cookie.
  * @fsl_chan : Freescale DMA channel
  */
@@ -883,6 +1121,7 @@ static int __devinit fsl_dma_chan_probe(struct fsl_dma_device *fdev,
                new_fsl_chan->toggle_ext_start = fsl_chan_toggle_ext_start;
                new_fsl_chan->set_src_loop_size = fsl_chan_set_src_loop_size;
                new_fsl_chan->set_dest_loop_size = fsl_chan_set_dest_loop_size;
+               new_fsl_chan->set_request_count = fsl_chan_set_request_count;
        }
 
        spin_lock_init(&new_fsl_chan->desc_lock);
@@ -962,12 +1201,15 @@ static int __devinit of_fsl_dma_probe(struct of_device *dev,
 
        dma_cap_set(DMA_MEMCPY, fdev->common.cap_mask);
        dma_cap_set(DMA_INTERRUPT, fdev->common.cap_mask);
+       dma_cap_set(DMA_SLAVE, fdev->common.cap_mask);
        fdev->common.device_alloc_chan_resources = fsl_dma_alloc_chan_resources;
        fdev->common.device_free_chan_resources = fsl_dma_free_chan_resources;
        fdev->common.device_prep_dma_interrupt = fsl_dma_prep_interrupt;
        fdev->common.device_prep_dma_memcpy = fsl_dma_prep_memcpy;
        fdev->common.device_is_tx_complete = fsl_dma_is_complete;
        fdev->common.device_issue_pending = fsl_dma_memcpy_issue_pending;
+       fdev->common.device_prep_slave_sg = fsl_dma_prep_slave_sg;
+       fdev->common.device_terminate_all = fsl_dma_device_terminate_all;
        fdev->common.dev = &dev->dev;
 
        fdev->irq = irq_of_parse_and_map(dev->node, 0);
index dc7f268..0df14cb 100644 (file)
@@ -90,6 +90,7 @@ struct fsl_dma_ld_hw {
 struct fsl_desc_sw {
        struct fsl_dma_ld_hw hw;
        struct list_head node;
+       struct list_head tx_list;
        struct dma_async_tx_descriptor async_tx;
        struct list_head *ld;
        void *priv;
@@ -143,10 +144,11 @@ struct fsl_dma_chan {
        struct tasklet_struct tasklet;
        u32 feature;
 
-       void (*toggle_ext_pause)(struct fsl_dma_chan *fsl_chan, int size);
+       void (*toggle_ext_pause)(struct fsl_dma_chan *fsl_chan, int enable);
        void (*toggle_ext_start)(struct fsl_dma_chan *fsl_chan, int enable);
        void (*set_src_loop_size)(struct fsl_dma_chan *fsl_chan, int size);
        void (*set_dest_loop_size)(struct fsl_dma_chan *fsl_chan, int size);
+       void (*set_request_count)(struct fsl_dma_chan *fsl_chan, int size);
 };
 
 #define to_fsl_chan(chan) container_of(chan, struct fsl_dma_chan, common)
diff --git a/drivers/dma/ioat.c b/drivers/dma/ioat.c
deleted file mode 100644 (file)
index 2225bb6..0000000
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * Intel I/OAT DMA Linux driver
- * Copyright(c) 2007 - 2009 Intel Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * The full GNU General Public License is included in this distribution in
- * the file called "COPYING".
- *
- */
-
-/*
- * This driver supports an Intel I/OAT DMA engine, which does asynchronous
- * copy operations.
- */
-
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/pci.h>
-#include <linux/interrupt.h>
-#include <linux/dca.h>
-#include "ioatdma.h"
-#include "ioatdma_registers.h"
-#include "ioatdma_hw.h"
-
-MODULE_VERSION(IOAT_DMA_VERSION);
-MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Intel Corporation");
-
-static struct pci_device_id ioat_pci_tbl[] = {
-       /* I/OAT v1 platforms */
-       { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT) },
-       { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_CNB)  },
-       { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SCNB) },
-       { PCI_DEVICE(PCI_VENDOR_ID_UNISYS, PCI_DEVICE_ID_UNISYS_DMA_DIRECTOR) },
-
-       /* I/OAT v2 platforms */
-       { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB) },
-
-       /* I/OAT v3 platforms */
-       { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG0) },
-       { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG1) },
-       { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG2) },
-       { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG3) },
-       { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG4) },
-       { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG5) },
-       { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG6) },
-       { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_TBG7) },
-       { 0, }
-};
-
-struct ioat_device {
-       struct pci_dev          *pdev;
-       void __iomem            *iobase;
-       struct ioatdma_device   *dma;
-       struct dca_provider     *dca;
-};
-
-static int __devinit ioat_probe(struct pci_dev *pdev,
-                               const struct pci_device_id *id);
-static void __devexit ioat_remove(struct pci_dev *pdev);
-
-static int ioat_dca_enabled = 1;
-module_param(ioat_dca_enabled, int, 0644);
-MODULE_PARM_DESC(ioat_dca_enabled, "control support of dca service (default: 1)");
-
-static struct pci_driver ioat_pci_driver = {
-       .name           = "ioatdma",
-       .id_table       = ioat_pci_tbl,
-       .probe          = ioat_probe,
-       .remove         = __devexit_p(ioat_remove),
-};
-
-static int __devinit ioat_probe(struct pci_dev *pdev,
-                               const struct pci_device_id *id)
-{
-       void __iomem *iobase;
-       struct ioat_device *device;
-       unsigned long mmio_start, mmio_len;
-       int err;
-
-       err = pci_enable_device(pdev);
-       if (err)
-               goto err_enable_device;
-
-       err = pci_request_regions(pdev, ioat_pci_driver.name);
-       if (err)
-               goto err_request_regions;
-
-       err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
-       if (err)
-               err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
-       if (err)
-               goto err_set_dma_mask;
-
-       err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
-       if (err)
-               err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
-       if (err)
-               goto err_set_dma_mask;
-
-       mmio_start = pci_resource_start(pdev, 0);
-       mmio_len = pci_resource_len(pdev, 0);
-       iobase = ioremap(mmio_start, mmio_len);
-       if (!iobase) {
-               err = -ENOMEM;
-               goto err_ioremap;
-       }
-
-       device = kzalloc(sizeof(*device), GFP_KERNEL);
-       if (!device) {
-               err = -ENOMEM;
-               goto err_kzalloc;
-       }
-       device->pdev = pdev;
-       pci_set_drvdata(pdev, device);
-       device->iobase = iobase;
-
-       pci_set_master(pdev);
-
-       switch (readb(iobase + IOAT_VER_OFFSET)) {
-       case IOAT_VER_1_2:
-               device->dma = ioat_dma_probe(pdev, iobase);
-               if (device->dma && ioat_dca_enabled)
-                       device->dca = ioat_dca_init(pdev, iobase);
-               break;
-       case IOAT_VER_2_0:
-               device->dma = ioat_dma_probe(pdev, iobase);
-               if (device->dma && ioat_dca_enabled)
-                       device->dca = ioat2_dca_init(pdev, iobase);
-               break;
-       case IOAT_VER_3_0:
-               device->dma = ioat_dma_probe(pdev, iobase);
-               if (device->dma && ioat_dca_enabled)
-                       device->dca = ioat3_dca_init(pdev, iobase);
-               break;
-       default:
-               err = -ENODEV;
-               break;
-       }
-       if (!device->dma)
-               err = -ENODEV;
-
-       if (err)
-               goto err_version;
-
-       return 0;
-
-err_version:
-       kfree(device);
-err_kzalloc:
-       iounmap(iobase);
-err_ioremap:
-err_set_dma_mask:
-       pci_release_regions(pdev);
-       pci_disable_device(pdev);
-err_request_regions:
-err_enable_device:
-       return err;
-}
-
-static void __devexit ioat_remove(struct pci_dev *pdev)
-{
-       struct ioat_device *device = pci_get_drvdata(pdev);
-
-       dev_err(&pdev->dev, "Removing dma and dca services\n");
-       if (device->dca) {
-               unregister_dca_provider(device->dca);
-               free_dca_provider(device->dca);
-               device->dca = NULL;
-       }
-
-       if (device->dma) {
-               ioat_dma_remove(device->dma);
-               device->dma = NULL;
-       }
-
-       kfree(device);
-}
-
-static int __init ioat_init_module(void)
-{
-       return pci_register_driver(&ioat_pci_driver);
-}
-module_init(ioat_init_module);
-
-static void __exit ioat_exit_module(void)
-{
-       pci_unregister_driver(&ioat_pci_driver);
-}
-module_exit(ioat_exit_module);
diff --git a/drivers/dma/ioat/Makefile b/drivers/dma/ioat/Makefile
new file mode 100644 (file)
index 0000000..8997d3f
--- /dev/null
@@ -0,0 +1,2 @@
+obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o
+ioatdma-objs := pci.o dma.o dma_v2.o dma_v3.o dca.o
similarity index 98%
rename from drivers/dma/ioat_dca.c
rename to drivers/dma/ioat/dca.c
index c012a1e..69d0261 100644 (file)
@@ -33,8 +33,8 @@
 #define cpu_physical_id(cpu) (cpuid_ebx(1) >> 24)
 #endif
 
-#include "ioatdma.h"
-#include "ioatdma_registers.h"
+#include "dma.h"
+#include "registers.h"
 
 /*
  * Bit 7 of a tag map entry is the "valid" bit, if it is set then bits 0:6
@@ -242,7 +242,8 @@ static struct dca_ops ioat_dca_ops = {
 };
 
 
-struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase)
+struct dca_provider * __devinit
+ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase)
 {
        struct dca_provider *dca;
        struct ioat_dca_priv *ioatdca;
@@ -407,7 +408,8 @@ static int ioat2_dca_count_dca_slots(void __iomem *iobase, u16 dca_offset)
        return slots;
 }
 
-struct dca_provider *ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase)
+struct dca_provider * __devinit
+ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase)
 {
        struct dca_provider *dca;
        struct ioat_dca_priv *ioatdca;
@@ -602,7 +604,8 @@ static int ioat3_dca_count_dca_slots(void *iobase, u16 dca_offset)
        return slots;
 }
 
-struct dca_provider *ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase)
+struct dca_provider * __devinit
+ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase)
 {
        struct dca_provider *dca;
        struct ioat_dca_priv *ioatdca;
diff --git a/drivers/dma/ioat/dma.c b/drivers/dma/ioat/dma.c
new file mode 100644 (file)
index 0000000..c524d36
--- /dev/null
@@ -0,0 +1,1238 @@
+/*
+ * Intel I/OAT DMA Linux driver
+ * Copyright(c) 2004 - 2009 Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ *
+ */
+
+/*
+ * This driver supports an Intel I/OAT DMA engine, which does asynchronous
+ * copy operations.
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/interrupt.h>
+#include <linux/dmaengine.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/workqueue.h>
+#include <linux/i7300_idle.h>
+#include "dma.h"
+#include "registers.h"
+#include "hw.h"
+
+int ioat_pending_level = 4;
+module_param(ioat_pending_level, int, 0644);
+MODULE_PARM_DESC(ioat_pending_level,
+                "high-water mark for pushing ioat descriptors (default: 4)");
+
+/* internal functions */
+static void ioat1_cleanup(struct ioat_dma_chan *ioat);
+static void ioat1_dma_start_null_desc(struct ioat_dma_chan *ioat);
+
+/**
+ * ioat_dma_do_interrupt - handler used for single vector interrupt mode
+ * @irq: interrupt id
+ * @data: interrupt data
+ */
+static irqreturn_t ioat_dma_do_interrupt(int irq, void *data)
+{
+       struct ioatdma_device *instance = data;
+       struct ioat_chan_common *chan;
+       unsigned long attnstatus;
+       int bit;
+       u8 intrctrl;
+
+       intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET);
+
+       if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
+               return IRQ_NONE;
+
+       if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
+               writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
+               return IRQ_NONE;
+       }
+
+       attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);
+       for_each_bit(bit, &attnstatus, BITS_PER_LONG) {
+               chan = ioat_chan_by_index(instance, bit);
+               tasklet_schedule(&chan->cleanup_task);
+       }
+
+       writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
+       return IRQ_HANDLED;
+}
+
+/**
+ * ioat_dma_do_interrupt_msix - handler used for vector-per-channel interrupt mode
+ * @irq: interrupt id
+ * @data: interrupt data
+ */
+static irqreturn_t ioat_dma_do_interrupt_msix(int irq, void *data)
+{
+       struct ioat_chan_common *chan = data;
+
+       tasklet_schedule(&chan->cleanup_task);
+
+       return IRQ_HANDLED;
+}
+
+static void ioat1_cleanup_tasklet(unsigned long data);
+
+/* common channel initialization */
+void ioat_init_channel(struct ioatdma_device *device,
+                      struct ioat_chan_common *chan, int idx,
+                      void (*timer_fn)(unsigned long),
+                      void (*tasklet)(unsigned long),
+                      unsigned long ioat)
+{
+       struct dma_device *dma = &device->common;
+
+       chan->device = device;
+       chan->reg_base = device->reg_base + (0x80 * (idx + 1));
+       spin_lock_init(&chan->cleanup_lock);
+       chan->common.device = dma;
+       list_add_tail(&chan->common.device_node, &dma->channels);
+       device->idx[idx] = chan;
+       init_timer(&chan->timer);
+       chan->timer.function = timer_fn;
+       chan->timer.data = ioat;
+       tasklet_init(&chan->cleanup_task, tasklet, ioat);
+       tasklet_disable(&chan->cleanup_task);
+}
+
+static void ioat1_timer_event(unsigned long data);
+
+/**
+ * ioat1_dma_enumerate_channels - find and initialize the device's channels
+ * @device: the device to be enumerated
+ */
+static int ioat1_enumerate_channels(struct ioatdma_device *device)
+{
+       u8 xfercap_scale;
+       u32 xfercap;
+       int i;
+       struct ioat_dma_chan *ioat;
+       struct device *dev = &device->pdev->dev;
+       struct dma_device *dma = &device->common;
+
+       INIT_LIST_HEAD(&dma->channels);
+       dma->chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
+       dma->chancnt &= 0x1f; /* bits [4:0] valid */
+       if (dma->chancnt > ARRAY_SIZE(device->idx)) {
+               dev_warn(dev, "(%d) exceeds max supported channels (%zu)\n",
+                        dma->chancnt, ARRAY_SIZE(device->idx));
+               dma->chancnt = ARRAY_SIZE(device->idx);
+       }
+       xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
+       xfercap_scale &= 0x1f; /* bits [4:0] valid */
+       xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale));
+       dev_dbg(dev, "%s: xfercap = %d\n", __func__, xfercap);
+
+#ifdef  CONFIG_I7300_IDLE_IOAT_CHANNEL
+       if (i7300_idle_platform_probe(NULL, NULL, 1) == 0)
+               dma->chancnt--;
+#endif
+       for (i = 0; i < dma->chancnt; i++) {
+               ioat = devm_kzalloc(dev, sizeof(*ioat), GFP_KERNEL);
+               if (!ioat)
+                       break;
+
+               ioat_init_channel(device, &ioat->base, i,
+                                 ioat1_timer_event,
+                                 ioat1_cleanup_tasklet,
+                                 (unsigned long) ioat);
+               ioat->xfercap = xfercap;
+               spin_lock_init(&ioat->desc_lock);
+               INIT_LIST_HEAD(&ioat->free_desc);
+               INIT_LIST_HEAD(&ioat->used_desc);
+       }
+       dma->chancnt = i;
+       return i;
+}
+
+/**
+ * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended
+ *                                 descriptors to hw
+ * @chan: DMA channel handle
+ */
+static inline void
+__ioat1_dma_memcpy_issue_pending(struct ioat_dma_chan *ioat)
+{
+       void __iomem *reg_base = ioat->base.reg_base;
+
+       dev_dbg(to_dev(&ioat->base), "%s: pending: %d\n",
+               __func__, ioat->pending);
+       ioat->pending = 0;
+       writeb(IOAT_CHANCMD_APPEND, reg_base + IOAT1_CHANCMD_OFFSET);
+}
+
+static void ioat1_dma_memcpy_issue_pending(struct dma_chan *chan)
+{
+       struct ioat_dma_chan *ioat = to_ioat_chan(chan);
+
+       if (ioat->pending > 0) {
+               spin_lock_bh(&ioat->desc_lock);
+               __ioat1_dma_memcpy_issue_pending(ioat);
+               spin_unlock_bh(&ioat->desc_lock);
+       }
+}
+
+/**
+ * ioat1_reset_channel - restart a channel
+ * @ioat: IOAT DMA channel handle
+ */
+static void ioat1_reset_channel(struct ioat_dma_chan *ioat)
+{
+       struct ioat_chan_common *chan = &ioat->base;
+       void __iomem *reg_base = chan->reg_base;
+       u32 chansts, chanerr;
+
+       dev_warn(to_dev(chan), "reset\n");
+       chanerr = readl(reg_base + IOAT_CHANERR_OFFSET);
+       chansts = *chan->completion & IOAT_CHANSTS_STATUS;
+       if (chanerr) {
+               dev_err(to_dev(chan),
+                       "chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n",
+                       chan_num(chan), chansts, chanerr);
+               writel(chanerr, reg_base + IOAT_CHANERR_OFFSET);
+       }
+
+       /*
+        * whack it upside the head with a reset
+        * and wait for things to settle out.
+        * force the pending count to a really big negative
+        * to make sure no one forces an issue_pending
+        * while we're waiting.
+        */
+
+       ioat->pending = INT_MIN;
+       writeb(IOAT_CHANCMD_RESET,
+              reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));
+       set_bit(IOAT_RESET_PENDING, &chan->state);
+       mod_timer(&chan->timer, jiffies + RESET_DELAY);
+}
+
+static dma_cookie_t ioat1_tx_submit(struct dma_async_tx_descriptor *tx)
+{
+       struct dma_chan *c = tx->chan;
+       struct ioat_dma_chan *ioat = to_ioat_chan(c);
+       struct ioat_desc_sw *desc = tx_to_ioat_desc(tx);
+       struct ioat_chan_common *chan = &ioat->base;
+       struct ioat_desc_sw *first;
+       struct ioat_desc_sw *chain_tail;
+       dma_cookie_t cookie;
+
+       spin_lock_bh(&ioat->desc_lock);
+       /* cookie incr and addition to used_list must be atomic */
+       cookie = c->cookie;
+       cookie++;
+       if (cookie < 0)
+               cookie = 1;
+       c->cookie = cookie;
+       tx->cookie = cookie;
+       dev_dbg(to_dev(&ioat->base), "%s: cookie: %d\n", __func__, cookie);
+
+       /* write address into NextDescriptor field of last desc in chain */
+       first = to_ioat_desc(desc->tx_list.next);
+       chain_tail = to_ioat_desc(ioat->used_desc.prev);
+       /* make descriptor updates globally visible before chaining */
+       wmb();
+       chain_tail->hw->next = first->txd.phys;
+       list_splice_tail_init(&desc->tx_list, &ioat->used_desc);
+       dump_desc_dbg(ioat, chain_tail);
+       dump_desc_dbg(ioat, first);
+
+       if (!test_and_set_bit(IOAT_COMPLETION_PENDING, &chan->state))
+               mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+
+       ioat->active += desc->hw->tx_cnt;
+       ioat->pending += desc->hw->tx_cnt;
+       if (ioat->pending >= ioat_pending_level)
+               __ioat1_dma_memcpy_issue_pending(ioat);
+       spin_unlock_bh(&ioat->desc_lock);
+
+       return cookie;
+}
+
+/**
+ * ioat_dma_alloc_descriptor - allocate and return a sw and hw descriptor pair
+ * @ioat: the channel supplying the memory pool for the descriptors
+ * @flags: allocation flags
+ */
+static struct ioat_desc_sw *
+ioat_dma_alloc_descriptor(struct ioat_dma_chan *ioat, gfp_t flags)
+{
+       struct ioat_dma_descriptor *desc;
+       struct ioat_desc_sw *desc_sw;
+       struct ioatdma_device *ioatdma_device;
+       dma_addr_t phys;
+
+       ioatdma_device = ioat->base.device;
+       desc = pci_pool_alloc(ioatdma_device->dma_pool, flags, &phys);
+       if (unlikely(!desc))
+               return NULL;
+
+       desc_sw = kzalloc(sizeof(*desc_sw), flags);
+       if (unlikely(!desc_sw)) {
+               pci_pool_free(ioatdma_device->dma_pool, desc, phys);
+               return NULL;
+       }
+
+       memset(desc, 0, sizeof(*desc));
+
+       INIT_LIST_HEAD(&desc_sw->tx_list);
+       dma_async_tx_descriptor_init(&desc_sw->txd, &ioat->base.common);
+       desc_sw->txd.tx_submit = ioat1_tx_submit;
+       desc_sw->hw = desc;
+       desc_sw->txd.phys = phys;
+       set_desc_id(desc_sw, -1);
+
+       return desc_sw;
+}
+
+static int ioat_initial_desc_count = 256;
+module_param(ioat_initial_desc_count, int, 0644);
+MODULE_PARM_DESC(ioat_initial_desc_count,
+                "ioat1: initial descriptors per channel (default: 256)");
+/**
+ * ioat1_dma_alloc_chan_resources - returns the number of allocated descriptors
+ * @chan: the channel to be filled out
+ */
+static int ioat1_dma_alloc_chan_resources(struct dma_chan *c)
+{
+       struct ioat_dma_chan *ioat = to_ioat_chan(c);
+       struct ioat_chan_common *chan = &ioat->base;
+       struct ioat_desc_sw *desc;
+       u32 chanerr;
+       int i;
+       LIST_HEAD(tmp_list);
+
+       /* have we already been set up? */
+       if (!list_empty(&ioat->free_desc))
+               return ioat->desccount;
+
+       /* Setup register to interrupt and write completion status on error */
+       writew(IOAT_CHANCTRL_RUN, chan->reg_base + IOAT_CHANCTRL_OFFSET);
+
+       chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
+       if (chanerr) {
+               dev_err(to_dev(chan), "CHANERR = %x, clearing\n", chanerr);
+               writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET);
+       }
+
+       /* Allocate descriptors */
+       for (i = 0; i < ioat_initial_desc_count; i++) {
+               desc = ioat_dma_alloc_descriptor(ioat, GFP_KERNEL);
+               if (!desc) {
+                       dev_err(to_dev(chan), "Only %d initial descriptors\n", i);
+                       break;
+               }
+               set_desc_id(desc, i);
+               list_add_tail(&desc->node, &tmp_list);
+       }
+       spin_lock_bh(&ioat->desc_lock);
+       ioat->desccount = i;
+       list_splice(&tmp_list, &ioat->free_desc);
+       spin_unlock_bh(&ioat->desc_lock);
+
+       /* allocate a completion writeback area */
+       /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
+       chan->completion = pci_pool_alloc(chan->device->completion_pool,
+                                         GFP_KERNEL, &chan->completion_dma);
+       memset(chan->completion, 0, sizeof(*chan->completion));
+       writel(((u64) chan->completion_dma) & 0x00000000FFFFFFFF,
+              chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
+       writel(((u64) chan->completion_dma) >> 32,
+              chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
+
+       tasklet_enable(&chan->cleanup_task);
+       ioat1_dma_start_null_desc(ioat);  /* give chain to dma device */
+       dev_dbg(to_dev(chan), "%s: allocated %d descriptors\n",
+               __func__, ioat->desccount);
+       return ioat->desccount;
+}
+
+/**
+ * ioat1_dma_free_chan_resources - release all the descriptors
+ * @chan: the channel to be cleaned
+ */
+static void ioat1_dma_free_chan_resources(struct dma_chan *c)
+{
+       struct ioat_dma_chan *ioat = to_ioat_chan(c);
+       struct ioat_chan_common *chan = &ioat->base;
+       struct ioatdma_device *ioatdma_device = chan->device;
+       struct ioat_desc_sw *desc, *_desc;
+       int in_use_descs = 0;
+
+       /* Before freeing channel resources first check
+        * if they have been previously allocated for this channel.
+        */
+       if (ioat->desccount == 0)
+               return;
+
+       tasklet_disable(&chan->cleanup_task);
+       del_timer_sync(&chan->timer);
+       ioat1_cleanup(ioat);
+
+       /* Delay 100ms after reset to allow internal DMA logic to quiesce
+        * before removing DMA descriptor resources.
+        */
+       writeb(IOAT_CHANCMD_RESET,
+              chan->reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));
+       mdelay(100);
+
+       spin_lock_bh(&ioat->desc_lock);
+       list_for_each_entry_safe(desc, _desc, &ioat->used_desc, node) {
+               dev_dbg(to_dev(chan), "%s: freeing %d from used list\n",
+                       __func__, desc_id(desc));
+               dump_desc_dbg(ioat, desc);
+               in_use_descs++;
+               list_del(&desc->node);
+               pci_pool_free(ioatdma_device->dma_pool, desc->hw,
+                             desc->txd.phys);
+               kfree(desc);
+       }
+       list_for_each_entry_safe(desc, _desc,
+                                &ioat->free_desc, node) {
+               list_del(&desc->node);
+               pci_pool_free(ioatdma_device->dma_pool, desc->hw,
+                             desc->txd.phys);
+               kfree(desc);
+       }
+       spin_unlock_bh(&ioat->desc_lock);
+
+       pci_pool_free(ioatdma_device->completion_pool,
+                     chan->completion,
+                     chan->completion_dma);
+
+       /* one is ok since we left it on there on purpose */
+       if (in_use_descs > 1)
+               dev_err(to_dev(chan), "Freeing %d in use descriptors!\n",
+                       in_use_descs - 1);
+
+       chan->last_completion = 0;
+       chan->completion_dma = 0;
+       ioat->pending = 0;
+       ioat->desccount = 0;
+}
+
+/**
+ * ioat1_dma_get_next_descriptor - return the next available descriptor
+ * @ioat: IOAT DMA channel handle
+ *
+ * Gets the next descriptor from the chain, and must be called with the
+ * channel's desc_lock held.  Allocates more descriptors if the channel
+ * has run out.
+ */
+static struct ioat_desc_sw *
+ioat1_dma_get_next_descriptor(struct ioat_dma_chan *ioat)
+{
+       struct ioat_desc_sw *new;
+
+       if (!list_empty(&ioat->free_desc)) {
+               new = to_ioat_desc(ioat->free_desc.next);
+               list_del(&new->node);
+       } else {
+               /* try to get another desc */
+               new = ioat_dma_alloc_descriptor(ioat, GFP_ATOMIC);
+               if (!new) {
+                       dev_err(to_dev(&ioat->base), "alloc failed\n");
+                       return NULL;
+               }
+       }
+       dev_dbg(to_dev(&ioat->base), "%s: allocated: %d\n",
+               __func__, desc_id(new));
+       prefetch(new->hw);
+       return new;
+}
+
+static struct dma_async_tx_descriptor *
+ioat1_dma_prep_memcpy(struct dma_chan *c, dma_addr_t dma_dest,
+                     dma_addr_t dma_src, size_t len, unsigned long flags)
+{
+       struct ioat_dma_chan *ioat = to_ioat_chan(c);
+       struct ioat_desc_sw *desc;
+       size_t copy;
+       LIST_HEAD(chain);
+       dma_addr_t src = dma_src;
+       dma_addr_t dest = dma_dest;
+       size_t total_len = len;
+       struct ioat_dma_descriptor *hw = NULL;
+       int tx_cnt = 0;
+
+       spin_lock_bh(&ioat->desc_lock);
+       desc = ioat1_dma_get_next_descriptor(ioat);
+       do {
+               if (!desc)
+                       break;
+
+               tx_cnt++;
+               copy = min_t(size_t, len, ioat->xfercap);
+
+               hw = desc->hw;
+               hw->size = copy;
+               hw->ctl = 0;
+               hw->src_addr = src;
+               hw->dst_addr = dest;
+
+               list_add_tail(&desc->node, &chain);
+
+               len -= copy;
+               dest += copy;
+               src += copy;
+               if (len) {
+                       struct ioat_desc_sw *next;
+
+                       async_tx_ack(&desc->txd);
+                       next = ioat1_dma_get_next_descriptor(ioat);
+                       hw->next = next ? next->txd.phys : 0;
+                       dump_desc_dbg(ioat, desc);
+                       desc = next;
+               } else
+                       hw->next = 0;
+       } while (len);
+
+       if (!desc) {
+               struct ioat_chan_common *chan = &ioat->base;
+
+               dev_err(to_dev(chan),
+                       "chan%d - get_next_desc failed\n", chan_num(chan));
+               list_splice(&chain, &ioat->free_desc);
+               spin_unlock_bh(&ioat->desc_lock);
+               return NULL;
+       }
+       spin_unlock_bh(&ioat->desc_lock);
+
+       desc->txd.flags = flags;
+       desc->len = total_len;
+       list_splice(&chain, &desc->tx_list);
+       hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
+       hw->ctl_f.compl_write = 1;
+       hw->tx_cnt = tx_cnt;
+       dump_desc_dbg(ioat, desc);
+
+       return &desc->txd;
+}
+
+static void ioat1_cleanup_tasklet(unsigned long data)
+{
+       struct ioat_dma_chan *chan = (void *)data;
+
+       ioat1_cleanup(chan);
+       writew(IOAT_CHANCTRL_RUN, chan->base.reg_base + IOAT_CHANCTRL_OFFSET);
+}
+
+void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags,
+                   size_t len, struct ioat_dma_descriptor *hw)
+{
+       struct pci_dev *pdev = chan->device->pdev;
+       size_t offset = len - hw->size;
+
+       if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP))
+               ioat_unmap(pdev, hw->dst_addr - offset, len,
+                          PCI_DMA_FROMDEVICE, flags, 1);
+
+       if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP))
+               ioat_unmap(pdev, hw->src_addr - offset, len,
+                          PCI_DMA_TODEVICE, flags, 0);
+}
+
+unsigned long ioat_get_current_completion(struct ioat_chan_common *chan)
+{
+       unsigned long phys_complete;
+       u64 completion;
+
+       completion = *chan->completion;
+       phys_complete = ioat_chansts_to_addr(completion);
+
+       dev_dbg(to_dev(chan), "%s: phys_complete: %#llx\n", __func__,
+               (unsigned long long) phys_complete);
+
+       if (is_ioat_halted(completion)) {
+               u32 chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
+               dev_err(to_dev(chan), "Channel halted, chanerr = %x\n",
+                       chanerr);
+
+               /* TODO do something to salvage the situation */
+       }
+
+       return phys_complete;
+}
+
+bool ioat_cleanup_preamble(struct ioat_chan_common *chan,
+                          unsigned long *phys_complete)
+{
+       *phys_complete = ioat_get_current_completion(chan);
+       if (*phys_complete == chan->last_completion)
+               return false;
+       clear_bit(IOAT_COMPLETION_ACK, &chan->state);
+       mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+
+       return true;
+}
+
+static void __cleanup(struct ioat_dma_chan *ioat, unsigned long phys_complete)
+{
+       struct ioat_chan_common *chan = &ioat->base;
+       struct list_head *_desc, *n;
+       struct dma_async_tx_descriptor *tx;
+
+       dev_dbg(to_dev(chan), "%s: phys_complete: %lx\n",
+                __func__, phys_complete);
+       list_for_each_safe(_desc, n, &ioat->used_desc) {
+               struct ioat_desc_sw *desc;
+
+               prefetch(n);
+               desc = list_entry(_desc, typeof(*desc), node);
+               tx = &desc->txd;
+               /*
+                * Incoming DMA requests may use multiple descriptors,
+                * due to exceeding xfercap, perhaps. If so, only the
+                * last one will have a cookie, and require unmapping.
+                */
+               dump_desc_dbg(ioat, desc);
+               if (tx->cookie) {
+                       chan->completed_cookie = tx->cookie;
+                       tx->cookie = 0;
+                       ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw);
+                       ioat->active -= desc->hw->tx_cnt;
+                       if (tx->callback) {
+                               tx->callback(tx->callback_param);
+                               tx->callback = NULL;
+                       }
+               }
+
+               if (tx->phys != phys_complete) {
+                       /*
+                        * a completed entry, but not the last, so clean
+                        * up if the client is done with the descriptor
+                        */
+                       if (async_tx_test_ack(tx))
+                               list_move_tail(&desc->node, &ioat->free_desc);
+               } else {
+                       /*
+                        * last used desc. Do not remove, so we can
+                        * append from it.
+                        */
+
+                       /* if nothing else is pending, cancel the
+                        * completion timeout
+                        */
+                       if (n == &ioat->used_desc) {
+                               dev_dbg(to_dev(chan),
+                                       "%s cancel completion timeout\n",
+                                       __func__);
+                               clear_bit(IOAT_COMPLETION_PENDING, &chan->state);
+                       }
+
+                       /* TODO check status bits? */
+                       break;
+               }
+       }
+
+       chan->last_completion = phys_complete;
+}
+
+/**
+ * ioat1_cleanup - cleanup up finished descriptors
+ * @chan: ioat channel to be cleaned up
+ *
+ * To prevent lock contention we defer cleanup when the locks are
+ * contended with a terminal timeout that forces cleanup and catches
+ * completion notification errors.
+ */
+static void ioat1_cleanup(struct ioat_dma_chan *ioat)
+{
+       struct ioat_chan_common *chan = &ioat->base;
+       unsigned long phys_complete;
+
+       prefetch(chan->completion);
+
+       if (!spin_trylock_bh(&chan->cleanup_lock))
+               return;
+
+       if (!ioat_cleanup_preamble(chan, &phys_complete)) {
+               spin_unlock_bh(&chan->cleanup_lock);
+               return;
+       }
+
+       if (!spin_trylock_bh(&ioat->desc_lock)) {
+               spin_unlock_bh(&chan->cleanup_lock);
+               return;
+       }
+
+       __cleanup(ioat, phys_complete);
+
+       spin_unlock_bh(&ioat->desc_lock);
+       spin_unlock_bh(&chan->cleanup_lock);
+}
+
+static void ioat1_timer_event(unsigned long data)
+{
+       struct ioat_dma_chan *ioat = (void *) data;
+       struct ioat_chan_common *chan = &ioat->base;
+
+       dev_dbg(to_dev(chan), "%s: state: %lx\n", __func__, chan->state);
+
+       spin_lock_bh(&chan->cleanup_lock);
+       if (test_and_clear_bit(IOAT_RESET_PENDING, &chan->state)) {
+               struct ioat_desc_sw *desc;
+
+               spin_lock_bh(&ioat->desc_lock);
+
+               /* restart active descriptors */
+               desc = to_ioat_desc(ioat->used_desc.prev);
+               ioat_set_chainaddr(ioat, desc->txd.phys);
+               ioat_start(chan);
+
+               ioat->pending = 0;
+               set_bit(IOAT_COMPLETION_PENDING, &chan->state);
+               mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+               spin_unlock_bh(&ioat->desc_lock);
+       } else if (test_bit(IOAT_COMPLETION_PENDING, &chan->state)) {
+               unsigned long phys_complete;
+
+               spin_lock_bh(&ioat->desc_lock);
+               /* if we haven't made progress and we have already
+                * acknowledged a pending completion once, then be more
+                * forceful with a restart
+                */
+               if (ioat_cleanup_preamble(chan, &phys_complete))
+                       __cleanup(ioat, phys_complete);
+               else if (test_bit(IOAT_COMPLETION_ACK, &chan->state))
+                       ioat1_reset_channel(ioat);
+               else {
+                       u64 status = ioat_chansts(chan);
+
+                       /* manually update the last completion address */
+                       if (ioat_chansts_to_addr(status) != 0)
+                               *chan->completion = status;
+
+                       set_bit(IOAT_COMPLETION_ACK, &chan->state);
+                       mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+               }
+               spin_unlock_bh(&ioat->desc_lock);
+       }
+       spin_unlock_bh(&chan->cleanup_lock);
+}
+
+static enum dma_status
+ioat1_dma_is_complete(struct dma_chan *c, dma_cookie_t cookie,
+                     dma_cookie_t *done, dma_cookie_t *used)
+{
+       struct ioat_dma_chan *ioat = to_ioat_chan(c);
+
+       if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS)
+               return DMA_SUCCESS;
+
+       ioat1_cleanup(ioat);
+
+       return ioat_is_complete(c, cookie, done, used);
+}
+
+static void ioat1_dma_start_null_desc(struct ioat_dma_chan *ioat)
+{
+       struct ioat_chan_common *chan = &ioat->base;
+       struct ioat_desc_sw *desc;
+       struct ioat_dma_descriptor *hw;
+
+       spin_lock_bh(&ioat->desc_lock);
+
+       desc = ioat1_dma_get_next_descriptor(ioat);
+
+       if (!desc) {
+               dev_err(to_dev(chan),
+                       "Unable to start null desc - get next desc failed\n");
+               spin_unlock_bh(&ioat->desc_lock);
+               return;
+       }
+
+       hw = desc->hw;
+       hw->ctl = 0;
+       hw->ctl_f.null = 1;
+       hw->ctl_f.int_en = 1;
+       hw->ctl_f.compl_write = 1;
+       /* set size to non-zero value (channel returns error when size is 0) */
+       hw->size = NULL_DESC_BUFFER_SIZE;
+       hw->src_addr = 0;
+       hw->dst_addr = 0;
+       async_tx_ack(&desc->txd);
+       hw->next = 0;
+       list_add_tail(&desc->node, &ioat->used_desc);
+       dump_desc_dbg(ioat, desc);
+
+       ioat_set_chainaddr(ioat, desc->txd.phys);
+       ioat_start(chan);
+       spin_unlock_bh(&ioat->desc_lock);
+}
+
+/*
+ * Perform a IOAT transaction to verify the HW works.
+ */
+#define IOAT_TEST_SIZE 2000
+
+static void __devinit ioat_dma_test_callback(void *dma_async_param)
+{
+       struct completion *cmp = dma_async_param;
+
+       complete(cmp);
+}
+
+/**
+ * ioat_dma_self_test - Perform a IOAT transaction to verify the HW works.
+ * @device: device to be tested
+ */
+int __devinit ioat_dma_self_test(struct ioatdma_device *device)
+{
+       int i;
+       u8 *src;
+       u8 *dest;
+       struct dma_device *dma = &device->common;
+       struct device *dev = &device->pdev->dev;
+       struct dma_chan *dma_chan;
+       struct dma_async_tx_descriptor *tx;
+       dma_addr_t dma_dest, dma_src;
+       dma_cookie_t cookie;
+       int err = 0;
+       struct completion cmp;
+       unsigned long tmo;
+       unsigned long flags;
+
+       src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
+       if (!src)
+               return -ENOMEM;
+       dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, GFP_KERNEL);
+       if (!dest) {
+               kfree(src);
+               return -ENOMEM;
+       }
+
+       /* Fill in src buffer */
+       for (i = 0; i < IOAT_TEST_SIZE; i++)
+               src[i] = (u8)i;
+
+       /* Start copy, using first DMA channel */
+       dma_chan = container_of(dma->channels.next, struct dma_chan,
+                               device_node);
+       if (dma->device_alloc_chan_resources(dma_chan) < 1) {
+               dev_err(dev, "selftest cannot allocate chan resource\n");
+               err = -ENODEV;
+               goto out;
+       }
+
+       dma_src = dma_map_single(dev, src, IOAT_TEST_SIZE, DMA_TO_DEVICE);
+       dma_dest = dma_map_single(dev, dest, IOAT_TEST_SIZE, DMA_FROM_DEVICE);
+       flags = DMA_COMPL_SRC_UNMAP_SINGLE | DMA_COMPL_DEST_UNMAP_SINGLE |
+               DMA_PREP_INTERRUPT;
+       tx = device->common.device_prep_dma_memcpy(dma_chan, dma_dest, dma_src,
+                                                  IOAT_TEST_SIZE, flags);
+       if (!tx) {
+               dev_err(dev, "Self-test prep failed, disabling\n");
+               err = -ENODEV;
+               goto free_resources;
+       }
+
+       async_tx_ack(tx);
+       init_completion(&cmp);
+       tx->callback = ioat_dma_test_callback;
+       tx->callback_param = &cmp;
+       cookie = tx->tx_submit(tx);
+       if (cookie < 0) {
+               dev_err(dev, "Self-test setup failed, disabling\n");
+               err = -ENODEV;
+               goto free_resources;
+       }
+       dma->device_issue_pending(dma_chan);
+
+       tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
+
+       if (tmo == 0 ||
+           dma->device_is_tx_complete(dma_chan, cookie, NULL, NULL)
+                                       != DMA_SUCCESS) {
+               dev_err(dev, "Self-test copy timed out, disabling\n");
+               err = -ENODEV;
+               goto free_resources;
+       }
+       if (memcmp(src, dest, IOAT_TEST_SIZE)) {
+               dev_err(dev, "Self-test copy failed compare, disabling\n");
+               err = -ENODEV;
+               goto free_resources;
+       }
+
+free_resources:
+       dma->device_free_chan_resources(dma_chan);
+out:
+       kfree(src);
+       kfree(dest);
+       return err;
+}
+
+static char ioat_interrupt_style[32] = "msix";
+module_param_string(ioat_interrupt_style, ioat_interrupt_style,
+                   sizeof(ioat_interrupt_style), 0644);
+MODULE_PARM_DESC(ioat_interrupt_style,
+                "set ioat interrupt style: msix (default), "
+                "msix-single-vector, msi, intx)");
+
+/**
+ * ioat_dma_setup_interrupts - setup interrupt handler
+ * @device: ioat device
+ */
+static int ioat_dma_setup_interrupts(struct ioatdma_device *device)
+{
+       struct ioat_chan_common *chan;
+       struct pci_dev *pdev = device->pdev;
+       struct device *dev = &pdev->dev;
+       struct msix_entry *msix;
+       int i, j, msixcnt;
+       int err = -EINVAL;
+       u8 intrctrl = 0;
+
+       if (!strcmp(ioat_interrupt_style, "msix"))
+               goto msix;
+       if (!strcmp(ioat_interrupt_style, "msix-single-vector"))
+               goto msix_single_vector;
+       if (!strcmp(ioat_interrupt_style, "msi"))
+               goto msi;
+       if (!strcmp(ioat_interrupt_style, "intx"))
+               goto intx;
+       dev_err(dev, "invalid ioat_interrupt_style %s\n", ioat_interrupt_style);
+       goto err_no_irq;
+
+msix:
+       /* The number of MSI-X vectors should equal the number of channels */
+       msixcnt = device->common.chancnt;
+       for (i = 0; i < msixcnt; i++)
+               device->msix_entries[i].entry = i;
+
+       err = pci_enable_msix(pdev, device->msix_entries, msixcnt);
+       if (err < 0)
+               goto msi;
+       if (err > 0)
+               goto msix_single_vector;
+
+       for (i = 0; i < msixcnt; i++) {
+               msix = &device->msix_entries[i];
+               chan = ioat_chan_by_index(device, i);
+               err = devm_request_irq(dev, msix->vector,
+                                      ioat_dma_do_interrupt_msix, 0,
+                                      "ioat-msix", chan);
+               if (err) {
+                       for (j = 0; j < i; j++) {
+                               msix = &device->msix_entries[j];
+                               chan = ioat_chan_by_index(device, j);
+                               devm_free_irq(dev, msix->vector, chan);
+                       }
+                       goto msix_single_vector;
+               }
+       }
+       intrctrl |= IOAT_INTRCTRL_MSIX_VECTOR_CONTROL;
+       goto done;
+
+msix_single_vector:
+       msix = &device->msix_entries[0];
+       msix->entry = 0;
+       err = pci_enable_msix(pdev, device->msix_entries, 1);
+       if (err)
+               goto msi;
+
+       err = devm_request_irq(dev, msix->vector, ioat_dma_do_interrupt, 0,
+                              "ioat-msix", device);
+       if (err) {
+               pci_disable_msix(pdev);
+               goto msi;
+       }
+       goto done;
+
+msi:
+       err = pci_enable_msi(pdev);
+       if (err)
+               goto intx;
+
+       err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt, 0,
+                              "ioat-msi", device);
+       if (err) {
+               pci_disable_msi(pdev);
+               goto intx;
+       }
+       goto done;
+
+intx:
+       err = devm_request_irq(dev, pdev->irq, ioat_dma_do_interrupt,
+                              IRQF_SHARED, "ioat-intx", device);
+       if (err)
+               goto err_no_irq;
+
+done:
+       if (device->intr_quirk)
+               device->intr_quirk(device);
+       intrctrl |= IOAT_INTRCTRL_MASTER_INT_EN;
+       writeb(intrctrl, device->reg_base + IOAT_INTRCTRL_OFFSET);
+       return 0;
+
+err_no_irq:
+       /* Disable all interrupt generation */
+       writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
+       dev_err(dev, "no usable interrupts\n");
+       return err;
+}
+
+static void ioat_disable_interrupts(struct ioatdma_device *device)
+{
+       /* Disable all interrupt generation */
+       writeb(0, device->reg_base + IOAT_INTRCTRL_OFFSET);
+}
+
+int __devinit ioat_probe(struct ioatdma_device *device)
+{
+       int err = -ENODEV;
+       struct dma_device *dma = &device->common;
+       struct pci_dev *pdev = device->pdev;
+       struct device *dev = &pdev->dev;
+
+       /* DMA coherent memory pool for DMA descriptor allocations */
+       device->dma_pool = pci_pool_create("dma_desc_pool", pdev,
+                                          sizeof(struct ioat_dma_descriptor),
+                                          64, 0);
+       if (!device->dma_pool) {
+               err = -ENOMEM;
+               goto err_dma_pool;
+       }
+
+       device->completion_pool = pci_pool_create("completion_pool", pdev,
+                                                 sizeof(u64), SMP_CACHE_BYTES,
+                                                 SMP_CACHE_BYTES);
+
+       if (!device->completion_pool) {
+               err = -ENOMEM;
+               goto err_completion_pool;
+       }
+
+       device->enumerate_channels(device);
+
+       dma_cap_set(DMA_MEMCPY, dma->cap_mask);
+       dma->dev = &pdev->dev;
+
+       if (!dma->chancnt) {
+               dev_err(dev, "zero channels detected\n");
+               goto err_setup_interrupts;
+       }
+
+       err = ioat_dma_setup_interrupts(device);
+       if (err)
+               goto err_setup_interrupts;
+
+       err = device->self_test(device);
+       if (err)
+               goto err_self_test;
+
+       return 0;
+
+err_self_test:
+       ioat_disable_interrupts(device);
+err_setup_interrupts:
+       pci_pool_destroy(device->completion_pool);
+err_completion_pool:
+       pci_pool_destroy(device->dma_pool);
+err_dma_pool:
+       return err;
+}
+
+int __devinit ioat_register(struct ioatdma_device *device)
+{
+       int err = dma_async_device_register(&device->common);
+
+       if (err) {
+               ioat_disable_interrupts(device);
+               pci_pool_destroy(device->completion_pool);
+               pci_pool_destroy(device->dma_pool);
+       }
+
+       return err;
+}
+
+/* ioat1_intr_quirk - fix up dma ctrl register to enable / disable msi */
+static void ioat1_intr_quirk(struct ioatdma_device *device)
+{
+       struct pci_dev *pdev = device->pdev;
+       u32 dmactrl;
+
+       pci_read_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, &dmactrl);
+       if (pdev->msi_enabled)
+               dmactrl |= IOAT_PCI_DMACTRL_MSI_EN;
+       else
+               dmactrl &= ~IOAT_PCI_DMACTRL_MSI_EN;
+       pci_write_config_dword(pdev, IOAT_PCI_DMACTRL_OFFSET, dmactrl);
+}
+
+static ssize_t ring_size_show(struct dma_chan *c, char *page)
+{
+       struct ioat_dma_chan *ioat = to_ioat_chan(c);
+
+       return sprintf(page, "%d\n", ioat->desccount);
+}
+static struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size);
+
+static ssize_t ring_active_show(struct dma_chan *c, char *page)
+{
+       struct ioat_dma_chan *ioat = to_ioat_chan(c);
+
+       return sprintf(page, "%d\n", ioat->active);
+}
+static struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active);
+
+static ssize_t cap_show(struct dma_chan *c, char *page)
+{
+       struct dma_device *dma = c->device;
+
+       return sprintf(page, "copy%s%s%s%s%s%s\n",
+                      dma_has_cap(DMA_PQ, dma->cap_mask) ? " pq" : "",
+                      dma_has_cap(DMA_PQ_VAL, dma->cap_mask) ? " pq_val" : "",
+                      dma_has_cap(DMA_XOR, dma->cap_mask) ? " xor" : "",
+                      dma_has_cap(DMA_XOR_VAL, dma->cap_mask) ? " xor_val" : "",
+                      dma_has_cap(DMA_MEMSET, dma->cap_mask)  ? " fill" : "",
+                      dma_has_cap(DMA_INTERRUPT, dma->cap_mask) ? " intr" : "");
+
+}
+struct ioat_sysfs_entry ioat_cap_attr = __ATTR_RO(cap);
+
+static ssize_t version_show(struct dma_chan *c, char *page)
+{
+       struct dma_device *dma = c->device;
+       struct ioatdma_device *device = to_ioatdma_device(dma);
+
+       return sprintf(page, "%d.%d\n",
+                      device->version >> 4, device->version & 0xf);
+}
+struct ioat_sysfs_entry ioat_version_attr = __ATTR_RO(version);
+
+static struct attribute *ioat1_attrs[] = {
+       &ring_size_attr.attr,
+       &ring_active_attr.attr,
+       &ioat_cap_attr.attr,
+       &ioat_version_attr.attr,
+       NULL,
+};
+
+static ssize_t
+ioat_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
+{
+       struct ioat_sysfs_entry *entry;
+       struct ioat_chan_common *chan;
+
+       entry = container_of(attr, struct ioat_sysfs_entry, attr);
+       chan = container_of(kobj, struct ioat_chan_common, kobj);
+
+       if (!entry->show)
+               return -EIO;
+       return entry->show(&chan->common, page);
+}
+
+struct sysfs_ops ioat_sysfs_ops = {
+       .show   = ioat_attr_show,
+};
+
+static struct kobj_type ioat1_ktype = {
+       .sysfs_ops = &ioat_sysfs_ops,
+       .default_attrs = ioat1_attrs,
+};
+
+void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type)
+{
+       struct dma_device *dma = &device->common;
+       struct dma_chan *c;
+
+       list_for_each_entry(c, &dma->channels, device_node) {
+               struct ioat_chan_common *chan = to_chan_common(c);
+               struct kobject *parent = &c->dev->device.kobj;
+               int err;
+
+               err = kobject_init_and_add(&chan->kobj, type, parent, "quickdata");
+               if (err) {
+                       dev_warn(to_dev(chan),
+                                "sysfs init error (%d), continuing...\n", err);
+                       kobject_put(&chan->kobj);
+                       set_bit(IOAT_KOBJ_INIT_FAIL, &chan->state);
+               }
+       }
+}
+
+void ioat_kobject_del(struct ioatdma_device *device)
+{
+       struct dma_device *dma = &device->common;
+       struct dma_chan *c;
+
+       list_for_each_entry(c, &dma->channels, device_node) {
+               struct ioat_chan_common *chan = to_chan_common(c);
+
+               if (!test_bit(IOAT_KOBJ_INIT_FAIL, &chan->state)) {
+                       kobject_del(&chan->kobj);
+                       kobject_put(&chan->kobj);
+               }
+       }
+}
+
+int __devinit ioat1_dma_probe(struct ioatdma_device *device, int dca)
+{
+       struct pci_dev *pdev = device->pdev;
+       struct dma_device *dma;
+       int err;
+
+       device->intr_quirk = ioat1_intr_quirk;
+       device->enumerate_channels = ioat1_enumerate_channels;
+       device->self_test = ioat_dma_self_test;
+       dma = &device->common;
+       dma->device_prep_dma_memcpy = ioat1_dma_prep_memcpy;
+       dma->device_issue_pending = ioat1_dma_memcpy_issue_pending;
+       dma->device_alloc_chan_resources = ioat1_dma_alloc_chan_resources;
+       dma->device_free_chan_resources = ioat1_dma_free_chan_resources;
+       dma->device_is_tx_complete = ioat1_dma_is_complete;
+
+       err = ioat_probe(device);
+       if (err)
+               return err;
+       ioat_set_tcp_copy_break(4096);
+       err = ioat_register(device);
+       if (err)
+               return err;
+       ioat_kobject_add(device, &ioat1_ktype);
+
+       if (dca)
+               device->dca = ioat_dca_init(pdev, device->reg_base);
+
+       return err;
+}
+
+void __devexit ioat_dma_remove(struct ioatdma_device *device)
+{
+       struct dma_device *dma = &device->common;
+
+       ioat_disable_interrupts(device);
+
+       ioat_kobject_del(device);
+
+       dma_async_device_unregister(dma);
+
+       pci_pool_destroy(device->dma_pool);
+       pci_pool_destroy(device->completion_pool);
+
+       INIT_LIST_HEAD(&dma->channels);
+}
diff --git a/drivers/dma/ioat/dma.h b/drivers/dma/ioat/dma.h
new file mode 100644 (file)
index 0000000..c14fdfe
--- /dev/null
@@ -0,0 +1,337 @@
+/*
+ * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59
+ * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called COPYING.
+ */
+#ifndef IOATDMA_H
+#define IOATDMA_H
+
+#include <linux/dmaengine.h>
+#include "hw.h"
+#include "registers.h"
+#include <linux/init.h>
+#include <linux/dmapool.h>
+#include <linux/cache.h>
+#include <linux/pci_ids.h>
+#include <net/tcp.h>
+
+#define IOAT_DMA_VERSION  "4.00"
+
+#define IOAT_LOW_COMPLETION_MASK       0xffffffc0
+#define IOAT_DMA_DCA_ANY_CPU           ~0
+
+#define to_ioatdma_device(dev) container_of(dev, struct ioatdma_device, common)
+#define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node)
+#define tx_to_ioat_desc(tx) container_of(tx, struct ioat_desc_sw, txd)
+#define to_dev(ioat_chan) (&(ioat_chan)->device->pdev->dev)
+
+#define chan_num(ch) ((int)((ch)->reg_base - (ch)->device->reg_base) / 0x80)
+
+/*
+ * workaround for IOAT ver.3.0 null descriptor issue
+ * (channel returns error when size is 0)
+ */
+#define NULL_DESC_BUFFER_SIZE 1
+
+/**
+ * struct ioatdma_device - internal representation of a IOAT device
+ * @pdev: PCI-Express device
+ * @reg_base: MMIO register space base address
+ * @dma_pool: for allocating DMA descriptors
+ * @common: embedded struct dma_device
+ * @version: version of ioatdma device
+ * @msix_entries: irq handlers
+ * @idx: per channel data
+ * @dca: direct cache access context
+ * @intr_quirk: interrupt setup quirk (for ioat_v1 devices)
+ * @enumerate_channels: hw version specific channel enumeration
+ * @cleanup_tasklet: select between the v2 and v3 cleanup routines
+ * @timer_fn: select between the v2 and v3 timer watchdog routines
+ * @self_test: hardware version specific self test for each supported op type
+ *
+ * Note: the v3 cleanup routine supports raid operations
+ */
+struct ioatdma_device {
+       struct pci_dev *pdev;
+       void __iomem *reg_base;
+       struct pci_pool *dma_pool;
+       struct pci_pool *completion_pool;
+       struct dma_device common;
+       u8 version;
+       struct msix_entry msix_entries[4];
+       struct ioat_chan_common *idx[4];
+       struct dca_provider *dca;
+       void (*intr_quirk)(struct ioatdma_device *device);
+       int (*enumerate_channels)(struct ioatdma_device *device);
+       void (*cleanup_tasklet)(unsigned long data);
+       void (*timer_fn)(unsigned long data);
+       int (*self_test)(struct ioatdma_device *device);
+};
+
+struct ioat_chan_common {
+       struct dma_chan common;
+       void __iomem *reg_base;
+       unsigned long last_completion;
+       spinlock_t cleanup_lock;
+       dma_cookie_t completed_cookie;
+       unsigned long state;
+       #define IOAT_COMPLETION_PENDING 0
+       #define IOAT_COMPLETION_ACK 1
+       #define IOAT_RESET_PENDING 2
+       #define IOAT_KOBJ_INIT_FAIL 3
+       struct timer_list timer;
+       #define COMPLETION_TIMEOUT msecs_to_jiffies(100)
+       #define IDLE_TIMEOUT msecs_to_jiffies(2000)
+       #define RESET_DELAY msecs_to_jiffies(100)
+       struct ioatdma_device *device;
+       dma_addr_t completion_dma;
+       u64 *completion;
+       struct tasklet_struct cleanup_task;
+       struct kobject kobj;
+};
+
+struct ioat_sysfs_entry {
+       struct attribute attr;
+       ssize_t (*show)(struct dma_chan *, char *);
+};
+
+/**
+ * struct ioat_dma_chan - internal representation of a DMA channel
+ */
+struct ioat_dma_chan {
+       struct ioat_chan_common base;
+
+       size_t xfercap; /* XFERCAP register value expanded out */
+
+       spinlock_t desc_lock;
+       struct list_head free_desc;
+       struct list_head used_desc;
+
+       int pending;
+       u16 desccount;
+       u16 active;
+};
+
+static inline struct ioat_chan_common *to_chan_common(struct dma_chan *c)
+{
+       return container_of(c, struct ioat_chan_common, common);
+}
+
+static inline struct ioat_dma_chan *to_ioat_chan(struct dma_chan *c)
+{
+       struct ioat_chan_common *chan = to_chan_common(c);
+
+       return container_of(chan, struct ioat_dma_chan, base);
+}
+
+/**
+ * ioat_is_complete - poll the status of an ioat transaction
+ * @c: channel handle
+ * @cookie: transaction identifier
+ * @done: if set, updated with last completed transaction
+ * @used: if set, updated with last used transaction
+ */
+static inline enum dma_status
+ioat_is_complete(struct dma_chan *c, dma_cookie_t cookie,
+                dma_cookie_t *done, dma_cookie_t *used)
+{
+       struct ioat_chan_common *chan = to_chan_common(c);
+       dma_cookie_t last_used;
+       dma_cookie_t last_complete;
+
+       last_used = c->cookie;
+       last_complete = chan->completed_cookie;
+
+       if (done)
+               *done = last_complete;
+       if (used)
+               *used = last_used;
+
+       return dma_async_is_complete(cookie, last_complete, last_used);
+}
+
+/* wrapper around hardware descriptor format + additional software fields */
+
+/**
+ * struct ioat_desc_sw - wrapper around hardware descriptor
+ * @hw: hardware DMA descriptor (for memcpy)
+ * @node: this descriptor will either be on the free list,
+ *     or attached to a transaction list (tx_list)
+ * @txd: the generic software descriptor for all engines
+ * @id: identifier for debug
+ */
+struct ioat_desc_sw {
+       struct ioat_dma_descriptor *hw;
+       struct list_head node;
+       size_t len;
+       struct list_head tx_list;
+       struct dma_async_tx_descriptor txd;
+       #ifdef DEBUG
+       int id;
+       #endif
+};
+
+#ifdef DEBUG
+#define set_desc_id(desc, i) ((desc)->id = (i))
+#define desc_id(desc) ((desc)->id)
+#else
+#define set_desc_id(desc, i)
+#define desc_id(desc) (0)
+#endif
+
+static inline void
+__dump_desc_dbg(struct ioat_chan_common *chan, struct ioat_dma_descriptor *hw,
+               struct dma_async_tx_descriptor *tx, int id)
+{
+       struct device *dev = to_dev(chan);
+
+       dev_dbg(dev, "desc[%d]: (%#llx->%#llx) cookie: %d flags: %#x"
+               " ctl: %#x (op: %d int_en: %d compl: %d)\n", id,
+               (unsigned long long) tx->phys,
+               (unsigned long long) hw->next, tx->cookie, tx->flags,
+               hw->ctl, hw->ctl_f.op, hw->ctl_f.int_en, hw->ctl_f.compl_write);
+}
+
+#define dump_desc_dbg(c, d) \
+       ({ if (d) __dump_desc_dbg(&c->base, d->hw, &d->txd, desc_id(d)); 0; })
+
+static inline void ioat_set_tcp_copy_break(unsigned long copybreak)
+{
+       #ifdef CONFIG_NET_DMA
+       sysctl_tcp_dma_copybreak = copybreak;
+       #endif
+}
+
+static inline struct ioat_chan_common *
+ioat_chan_by_index(struct ioatdma_device *device, int index)
+{
+       return device->idx[index];
+}
+
+static inline u64 ioat_chansts(struct ioat_chan_common *chan)
+{
+       u8 ver = chan->device->version;
+       u64 status;
+       u32 status_lo;
+
+       /* We need to read the low address first as this causes the
+        * chipset to latch the upper bits for the subsequent read
+        */
+       status_lo = readl(chan->reg_base + IOAT_CHANSTS_OFFSET_LOW(ver));
+       status = readl(chan->reg_base + IOAT_CHANSTS_OFFSET_HIGH(ver));
+       status <<= 32;
+       status |= status_lo;
+
+       return status;
+}
+
+static inline void ioat_start(struct ioat_chan_common *chan)
+{
+       u8 ver = chan->device->version;
+
+       writeb(IOAT_CHANCMD_START, chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
+}
+
+static inline u64 ioat_chansts_to_addr(u64 status)
+{
+       return status & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
+}
+
+static inline u32 ioat_chanerr(struct ioat_chan_common *chan)
+{
+       return readl(chan->reg_base + IOAT_CHANERR_OFFSET);
+}
+
+static inline void ioat_suspend(struct ioat_chan_common *chan)
+{
+       u8 ver = chan->device->version;
+
+       writeb(IOAT_CHANCMD_SUSPEND, chan->reg_base + IOAT_CHANCMD_OFFSET(ver));
+}
+
+static inline void ioat_set_chainaddr(struct ioat_dma_chan *ioat, u64 addr)
+{
+       struct ioat_chan_common *chan = &ioat->base;
+
+       writel(addr & 0x00000000FFFFFFFF,
+              chan->reg_base + IOAT1_CHAINADDR_OFFSET_LOW);
+       writel(addr >> 32,
+              chan->reg_base + IOAT1_CHAINADDR_OFFSET_HIGH);
+}
+
+static inline bool is_ioat_active(unsigned long status)
+{
+       return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_ACTIVE);
+}
+
+static inline bool is_ioat_idle(unsigned long status)
+{
+       return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_DONE);
+}
+
+static inline bool is_ioat_halted(unsigned long status)
+{
+       return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_HALTED);
+}
+
+static inline bool is_ioat_suspended(unsigned long status)
+{
+       return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_SUSPENDED);
+}
+
+/* channel was fatally programmed */
+static inline bool is_ioat_bug(unsigned long err)
+{
+       return !!(err & (IOAT_CHANERR_SRC_ADDR_ERR|IOAT_CHANERR_DEST_ADDR_ERR|
+                        IOAT_CHANERR_NEXT_ADDR_ERR|IOAT_CHANERR_CONTROL_ERR|
+                        IOAT_CHANERR_LENGTH_ERR));
+}
+
+static inline void ioat_unmap(struct pci_dev *pdev, dma_addr_t addr, size_t len,
+                             int direction, enum dma_ctrl_flags flags, bool dst)
+{
+       if ((dst && (flags & DMA_COMPL_DEST_UNMAP_SINGLE)) ||
+           (!dst && (flags & DMA_COMPL_SRC_UNMAP_SINGLE)))
+               pci_unmap_single(pdev, addr, len, direction);
+       else
+               pci_unmap_page(pdev, addr, len, direction);
+}
+
+int __devinit ioat_probe(struct ioatdma_device *device);
+int __devinit ioat_register(struct ioatdma_device *device);
+int __devinit ioat1_dma_probe(struct ioatdma_device *dev, int dca);
+int __devinit ioat_dma_self_test(struct ioatdma_device *device);
+void __devexit ioat_dma_remove(struct ioatdma_device *device);
+struct dca_provider * __devinit ioat_dca_init(struct pci_dev *pdev,
+                                             void __iomem *iobase);
+unsigned long ioat_get_current_completion(struct ioat_chan_common *chan);
+void ioat_init_channel(struct ioatdma_device *device,
+                      struct ioat_chan_common *chan, int idx,
+                      void (*timer_fn)(unsigned long),
+                      void (*tasklet)(unsigned long),
+                      unsigned long ioat);
+void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags,
+                   size_t len, struct ioat_dma_descriptor *hw);
+bool ioat_cleanup_preamble(struct ioat_chan_common *chan,
+                          unsigned long *phys_complete);
+void ioat_kobject_add(struct ioatdma_device *device, struct kobj_type *type);
+void ioat_kobject_del(struct ioatdma_device *device);
+extern struct sysfs_ops ioat_sysfs_ops;
+extern struct ioat_sysfs_entry ioat_version_attr;
+extern struct ioat_sysfs_entry ioat_cap_attr;
+#endif /* IOATDMA_H */
diff --git a/drivers/dma/ioat/dma_v2.c b/drivers/dma/ioat/dma_v2.c
new file mode 100644 (file)
index 0000000..96ffab7
--- /dev/null
@@ -0,0 +1,871 @@
+/*
+ * Intel I/OAT DMA Linux driver
+ * Copyright(c) 2004 - 2009 Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ *
+ */
+
+/*
+ * This driver supports an Intel I/OAT DMA engine (versions >= 2), which
+ * does asynchronous data movement and checksumming operations.
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/interrupt.h>
+#include <linux/dmaengine.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/workqueue.h>
+#include <linux/i7300_idle.h>
+#include "dma.h"
+#include "dma_v2.h"
+#include "registers.h"
+#include "hw.h"
+
+int ioat_ring_alloc_order = 8;
+module_param(ioat_ring_alloc_order, int, 0644);
+MODULE_PARM_DESC(ioat_ring_alloc_order,
+                "ioat2+: allocate 2^n descriptors per channel"
+                " (default: 8 max: 16)");
+static int ioat_ring_max_alloc_order = IOAT_MAX_ORDER;
+module_param(ioat_ring_max_alloc_order, int, 0644);
+MODULE_PARM_DESC(ioat_ring_max_alloc_order,
+                "ioat2+: upper limit for ring size (default: 16)");
+
+void __ioat2_issue_pending(struct ioat2_dma_chan *ioat)
+{
+       void * __iomem reg_base = ioat->base.reg_base;
+
+       ioat->pending = 0;
+       ioat->dmacount += ioat2_ring_pending(ioat);
+       ioat->issued = ioat->head;
+       /* make descriptor updates globally visible before notifying channel */
+       wmb();
+       writew(ioat->dmacount, reg_base + IOAT_CHAN_DMACOUNT_OFFSET);
+       dev_dbg(to_dev(&ioat->base),
+               "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
+               __func__, ioat->head, ioat->tail, ioat->issued, ioat->dmacount);
+}
+
+void ioat2_issue_pending(struct dma_chan *chan)
+{
+       struct ioat2_dma_chan *ioat = to_ioat2_chan(chan);
+
+       spin_lock_bh(&ioat->ring_lock);
+       if (ioat->pending == 1)
+               __ioat2_issue_pending(ioat);
+       spin_unlock_bh(&ioat->ring_lock);
+}
+
+/**
+ * ioat2_update_pending - log pending descriptors
+ * @ioat: ioat2+ channel
+ *
+ * set pending to '1' unless pending is already set to '2', pending == 2
+ * indicates that submission is temporarily blocked due to an in-flight
+ * reset.  If we are already above the ioat_pending_level threshold then
+ * just issue pending.
+ *
+ * called with ring_lock held
+ */
+static void ioat2_update_pending(struct ioat2_dma_chan *ioat)
+{
+       if (unlikely(ioat->pending == 2))
+               return;
+       else if (ioat2_ring_pending(ioat) > ioat_pending_level)
+               __ioat2_issue_pending(ioat);
+       else
+               ioat->pending = 1;
+}
+
+static void __ioat2_start_null_desc(struct ioat2_dma_chan *ioat)
+{
+       struct ioat_ring_ent *desc;
+       struct ioat_dma_descriptor *hw;
+       int idx;
+
+       if (ioat2_ring_space(ioat) < 1) {
+               dev_err(to_dev(&ioat->base),
+                       "Unable to start null desc - ring full\n");
+               return;
+       }
+
+       dev_dbg(to_dev(&ioat->base), "%s: head: %#x tail: %#x issued: %#x\n",
+               __func__, ioat->head, ioat->tail, ioat->issued);
+       idx = ioat2_desc_alloc(ioat, 1);
+       desc = ioat2_get_ring_ent(ioat, idx);
+
+       hw = desc->hw;
+       hw->ctl = 0;
+       hw->ctl_f.null = 1;
+       hw->ctl_f.int_en = 1;
+       hw->ctl_f.compl_write = 1;
+       /* set size to non-zero value (channel returns error when size is 0) */
+       hw->size = NULL_DESC_BUFFER_SIZE;
+       hw->src_addr = 0;
+       hw->dst_addr = 0;
+       async_tx_ack(&desc->txd);
+       ioat2_set_chainaddr(ioat, desc->txd.phys);
+       dump_desc_dbg(ioat, desc);
+       __ioat2_issue_pending(ioat);
+}
+
+static void ioat2_start_null_desc(struct ioat2_dma_chan *ioat)
+{
+       spin_lock_bh(&ioat->ring_lock);
+       __ioat2_start_null_desc(ioat);
+       spin_unlock_bh(&ioat->ring_lock);
+}
+
+static void __cleanup(struct ioat2_dma_chan *ioat, unsigned long phys_complete)
+{
+       struct ioat_chan_common *chan = &ioat->base;
+       struct dma_async_tx_descriptor *tx;
+       struct ioat_ring_ent *desc;
+       bool seen_current = false;
+       u16 active;
+       int i;
+
+       dev_dbg(to_dev(chan), "%s: head: %#x tail: %#x issued: %#x\n",
+               __func__, ioat->head, ioat->tail, ioat->issued);
+
+       active = ioat2_ring_active(ioat);
+       for (i = 0; i < active && !seen_current; i++) {
+               prefetch(ioat2_get_ring_ent(ioat, ioat->tail + i + 1));
+               desc = ioat2_get_ring_ent(ioat, ioat->tail + i);
+               tx = &desc->txd;
+               dump_desc_dbg(ioat, desc);
+               if (tx->cookie) {
+                       ioat_dma_unmap(chan, tx->flags, desc->len, desc->hw);
+                       chan->completed_cookie = tx->cookie;
+                       tx->cookie = 0;
+                       if (tx->callback) {
+                               tx->callback(tx->callback_param);
+                               tx->callback = NULL;
+                       }
+               }
+
+               if (tx->phys == phys_complete)
+                       seen_current = true;
+       }
+       ioat->tail += i;
+       BUG_ON(!seen_current); /* no active descs have written a completion? */
+
+       chan->last_completion = phys_complete;
+       if (ioat->head == ioat->tail) {
+               dev_dbg(to_dev(chan), "%s: cancel completion timeout\n",
+                       __func__);
+               clear_bit(IOAT_COMPLETION_PENDING, &chan->state);
+               mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
+       }
+}
+
+/**
+ * ioat2_cleanup - clean finished descriptors (advance tail pointer)
+ * @chan: ioat channel to be cleaned up
+ */
+static void ioat2_cleanup(struct ioat2_dma_chan *ioat)
+{
+       struct ioat_chan_common *chan = &ioat->base;
+       unsigned long phys_complete;
+
+       prefetch(chan->completion);
+
+       if (!spin_trylock_bh(&chan->cleanup_lock))
+               return;
+
+       if (!ioat_cleanup_preamble(chan, &phys_complete)) {
+               spin_unlock_bh(&chan->cleanup_lock);
+               return;
+       }
+
+       if (!spin_trylock_bh(&ioat->ring_lock)) {
+               spin_unlock_bh(&chan->cleanup_lock);
+               return;
+       }
+
+       __cleanup(ioat, phys_complete);
+
+       spin_unlock_bh(&ioat->ring_lock);
+       spin_unlock_bh(&chan->cleanup_lock);
+}
+
+void ioat2_cleanup_tasklet(unsigned long data)
+{
+       struct ioat2_dma_chan *ioat = (void *) data;
+
+       ioat2_cleanup(ioat);
+       writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);
+}
+
+void __ioat2_restart_chan(struct ioat2_dma_chan *ioat)
+{
+       struct ioat_chan_common *chan = &ioat->base;
+
+       /* set the tail to be re-issued */
+       ioat->issued = ioat->tail;
+       ioat->dmacount = 0;
+       set_bit(IOAT_COMPLETION_PENDING, &chan->state);
+       mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+
+       dev_dbg(to_dev(chan),
+               "%s: head: %#x tail: %#x issued: %#x count: %#x\n",
+               __func__, ioat->head, ioat->tail, ioat->issued, ioat->dmacount);
+
+       if (ioat2_ring_pending(ioat)) {
+               struct ioat_ring_ent *desc;
+
+               desc = ioat2_get_ring_ent(ioat, ioat->tail);
+               ioat2_set_chainaddr(ioat, desc->txd.phys);
+               __ioat2_issue_pending(ioat);
+       } else
+               __ioat2_start_null_desc(ioat);
+}
+
+static void ioat2_restart_channel(struct ioat2_dma_chan *ioat)
+{
+       struct ioat_chan_common *chan = &ioat->base;
+       unsigned long phys_complete;
+       u32 status;
+
+       status = ioat_chansts(chan);
+       if (is_ioat_active(status) || is_ioat_idle(status))
+               ioat_suspend(chan);
+       while (is_ioat_active(status) || is_ioat_idle(status)) {
+               status = ioat_chansts(chan);
+               cpu_relax();
+       }
+
+       if (ioat_cleanup_preamble(chan, &phys_complete))
+               __cleanup(ioat, phys_complete);
+
+       __ioat2_restart_chan(ioat);
+}
+
+void ioat2_timer_event(unsigned long data)
+{
+       struct ioat2_dma_chan *ioat = (void *) data;
+       struct ioat_chan_common *chan = &ioat->base;
+
+       spin_lock_bh(&chan->cleanup_lock);
+       if (test_bit(IOAT_COMPLETION_PENDING, &chan->state)) {
+               unsigned long phys_complete;
+               u64 status;
+
+               spin_lock_bh(&ioat->ring_lock);
+               status = ioat_chansts(chan);
+
+               /* when halted due to errors check for channel
+                * programming errors before advancing the completion state
+                */
+               if (is_ioat_halted(status)) {
+                       u32 chanerr;
+
+                       chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
+                       BUG_ON(is_ioat_bug(chanerr));
+               }
+
+               /* if we haven't made progress and we have already
+                * acknowledged a pending completion once, then be more
+                * forceful with a restart
+                */
+               if (ioat_cleanup_preamble(chan, &phys_complete))
+                       __cleanup(ioat, phys_complete);
+               else if (test_bit(IOAT_COMPLETION_ACK, &chan->state))
+                       ioat2_restart_channel(ioat);
+               else {
+                       set_bit(IOAT_COMPLETION_ACK, &chan->state);
+                       mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+               }
+               spin_unlock_bh(&ioat->ring_lock);
+       } else {
+               u16 active;
+
+               /* if the ring is idle, empty, and oversized try to step
+                * down the size
+                */
+               spin_lock_bh(&ioat->ring_lock);
+               active = ioat2_ring_active(ioat);
+               if (active == 0 && ioat->alloc_order > ioat_get_alloc_order())
+                       reshape_ring(ioat, ioat->alloc_order-1);
+               spin_unlock_bh(&ioat->ring_lock);
+
+               /* keep shrinking until we get back to our minimum
+                * default size
+                */
+               if (ioat->alloc_order > ioat_get_alloc_order())
+                       mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
+       }
+       spin_unlock_bh(&chan->cleanup_lock);
+}
+
+/**
+ * ioat2_enumerate_channels - find and initialize the device's channels
+ * @device: the device to be enumerated
+ */
+int ioat2_enumerate_channels(struct ioatdma_device *device)
+{
+       struct ioat2_dma_chan *ioat;
+       struct device *dev = &device->pdev->dev;
+       struct dma_device *dma = &device->common;
+       u8 xfercap_log;
+       int i;
+
+       INIT_LIST_HEAD(&dma->channels);
+       dma->chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
+       dma->chancnt &= 0x1f; /* bits [4:0] valid */
+       if (dma->chancnt > ARRAY_SIZE(device->idx)) {
+               dev_warn(dev, "(%d) exceeds max supported channels (%zu)\n",
+                        dma->chancnt, ARRAY_SIZE(device->idx));
+               dma->chancnt = ARRAY_SIZE(device->idx);
+       }
+       xfercap_log = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
+       xfercap_log &= 0x1f; /* bits [4:0] valid */
+       if (xfercap_log == 0)
+               return 0;
+       dev_dbg(dev, "%s: xfercap = %d\n", __func__, 1 << xfercap_log);
+
+       /* FIXME which i/oat version is i7300? */
+#ifdef CONFIG_I7300_IDLE_IOAT_CHANNEL
+       if (i7300_idle_platform_probe(NULL, NULL, 1) == 0)
+               dma->chancnt--;
+#endif
+       for (i = 0; i < dma->chancnt; i++) {
+               ioat = devm_kzalloc(dev, sizeof(*ioat), GFP_KERNEL);
+               if (!ioat)
+                       break;
+
+               ioat_init_channel(device, &ioat->base, i,
+                                 device->timer_fn,
+                                 device->cleanup_tasklet,
+                                 (unsigned long) ioat);
+               ioat->xfercap_log = xfercap_log;
+               spin_lock_init(&ioat->ring_lock);
+       }
+       dma->chancnt = i;
+       return i;
+}
+
+static dma_cookie_t ioat2_tx_submit_unlock(struct dma_async_tx_descriptor *tx)
+{
+       struct dma_chan *c = tx->chan;
+       struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+       struct ioat_chan_common *chan = &ioat->base;
+       dma_cookie_t cookie = c->cookie;
+
+       cookie++;
+       if (cookie < 0)
+               cookie = 1;
+       tx->cookie = cookie;
+       c->cookie = cookie;
+       dev_dbg(to_dev(&ioat->base), "%s: cookie: %d\n", __func__, cookie);
+
+       if (!test_and_set_bit(IOAT_COMPLETION_PENDING, &chan->state))
+               mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+       ioat2_update_pending(ioat);
+       spin_unlock_bh(&ioat->ring_lock);
+
+       return cookie;
+}
+
+static struct ioat_ring_ent *ioat2_alloc_ring_ent(struct dma_chan *chan, gfp_t flags)
+{
+       struct ioat_dma_descriptor *hw;
+       struct ioat_ring_ent *desc;
+       struct ioatdma_device *dma;
+       dma_addr_t phys;
+
+       dma = to_ioatdma_device(chan->device);
+       hw = pci_pool_alloc(dma->dma_pool, flags, &phys);
+       if (!hw)
+               return NULL;
+       memset(hw, 0, sizeof(*hw));
+
+       desc = kmem_cache_alloc(ioat2_cache, flags);
+       if (!desc) {
+               pci_pool_free(dma->dma_pool, hw, phys);
+               return NULL;
+       }
+       memset(desc, 0, sizeof(*desc));
+
+       dma_async_tx_descriptor_init(&desc->txd, chan);
+       desc->txd.tx_submit = ioat2_tx_submit_unlock;
+       desc->hw = hw;
+       desc->txd.phys = phys;
+       return desc;
+}
+
+static void ioat2_free_ring_ent(struct ioat_ring_ent *desc, struct dma_chan *chan)
+{
+       struct ioatdma_device *dma;
+
+       dma = to_ioatdma_device(chan->device);
+       pci_pool_free(dma->dma_pool, desc->hw, desc->txd.phys);
+       kmem_cache_free(ioat2_cache, desc);
+}
+
+static struct ioat_ring_ent **ioat2_alloc_ring(struct dma_chan *c, int order, gfp_t flags)
+{
+       struct ioat_ring_ent **ring;
+       int descs = 1 << order;
+       int i;
+
+       if (order > ioat_get_max_alloc_order())
+               return NULL;
+
+       /* allocate the array to hold the software ring */
+       ring = kcalloc(descs, sizeof(*ring), flags);
+       if (!ring)
+               return NULL;
+       for (i = 0; i < descs; i++) {
+               ring[i] = ioat2_alloc_ring_ent(c, flags);
+               if (!ring[i]) {
+                       while (i--)
+                               ioat2_free_ring_ent(ring[i], c);
+                       kfree(ring);
+                       return NULL;
+               }
+               set_desc_id(ring[i], i);
+       }
+
+       /* link descs */
+       for (i = 0; i < descs-1; i++) {
+               struct ioat_ring_ent *next = ring[i+1];
+               struct ioat_dma_descriptor *hw = ring[i]->hw;
+
+               hw->next = next->txd.phys;
+       }
+       ring[i]->hw->next = ring[0]->txd.phys;
+
+       return ring;
+}
+
+/* ioat2_alloc_chan_resources - allocate/initialize ioat2 descriptor ring
+ * @chan: channel to be initialized
+ */
+int ioat2_alloc_chan_resources(struct dma_chan *c)
+{
+       struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+       struct ioat_chan_common *chan = &ioat->base;
+       struct ioat_ring_ent **ring;
+       u32 chanerr;
+       int order;
+
+       /* have we already been set up? */
+       if (ioat->ring)
+               return 1 << ioat->alloc_order;
+
+       /* Setup register to interrupt and write completion status on error */
+       writew(IOAT_CHANCTRL_RUN, chan->reg_base + IOAT_CHANCTRL_OFFSET);
+
+       chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
+       if (chanerr) {
+               dev_err(to_dev(chan), "CHANERR = %x, clearing\n", chanerr);
+               writel(chanerr, chan->reg_base + IOAT_CHANERR_OFFSET);
+       }
+
+       /* allocate a completion writeback area */
+       /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
+       chan->completion = pci_pool_alloc(chan->device->completion_pool,
+                                         GFP_KERNEL, &chan->completion_dma);
+       if (!chan->completion)
+               return -ENOMEM;
+
+       memset(chan->completion, 0, sizeof(*chan->completion));
+       writel(((u64) chan->completion_dma) & 0x00000000FFFFFFFF,
+              chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
+       writel(((u64) chan->completion_dma) >> 32,
+              chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
+
+       order = ioat_get_alloc_order();
+       ring = ioat2_alloc_ring(c, order, GFP_KERNEL);
+       if (!ring)
+               return -ENOMEM;
+
+       spin_lock_bh(&ioat->ring_lock);
+       ioat->ring = ring;
+       ioat->head = 0;
+       ioat->issued = 0;
+       ioat->tail = 0;
+       ioat->pending = 0;
+       ioat->alloc_order = order;
+       spin_unlock_bh(&ioat->ring_lock);
+
+       tasklet_enable(&chan->cleanup_task);
+       ioat2_start_null_desc(ioat);
+
+       return 1 << ioat->alloc_order;
+}
+
+bool reshape_ring(struct ioat2_dma_chan *ioat, int order)
+{
+       /* reshape differs from normal ring allocation in that we want
+        * to allocate a new software ring while only
+        * extending/truncating the hardware ring
+        */
+       struct ioat_chan_common *chan = &ioat->base;
+       struct dma_chan *c = &chan->common;
+       const u16 curr_size = ioat2_ring_mask(ioat) + 1;
+       const u16 active = ioat2_ring_active(ioat);
+       const u16 new_size = 1 << order;
+       struct ioat_ring_ent **ring;
+       u16 i;
+
+       if (order > ioat_get_max_alloc_order())
+               return false;
+
+       /* double check that we have at least 1 free descriptor */
+       if (active == curr_size)
+               return false;
+
+       /* when shrinking, verify that we can hold the current active
+        * set in the new ring
+        */
+       if (active >= new_size)
+               return false;
+
+       /* allocate the array to hold the software ring */
+       ring = kcalloc(new_size, sizeof(*ring), GFP_NOWAIT);
+       if (!ring)
+               return false;
+
+       /* allocate/trim descriptors as needed */
+       if (new_size > curr_size) {
+               /* copy current descriptors to the new ring */
+               for (i = 0; i < curr_size; i++) {
+                       u16 curr_idx = (ioat->tail+i) & (curr_size-1);
+                       u16 new_idx = (ioat->tail+i) & (new_size-1);
+
+                       ring[new_idx] = ioat->ring[curr_idx];
+                       set_desc_id(ring[new_idx], new_idx);
+               }
+
+               /* add new descriptors to the ring */
+               for (i = curr_size; i < new_size; i++) {
+                       u16 new_idx = (ioat->tail+i) & (new_size-1);
+
+                       ring[new_idx] = ioat2_alloc_ring_ent(c, GFP_NOWAIT);
+                       if (!ring[new_idx]) {
+                               while (i--) {
+                                       u16 new_idx = (ioat->tail+i) & (new_size-1);
+
+                                       ioat2_free_ring_ent(ring[new_idx], c);
+                               }
+                               kfree(ring);
+                               return false;
+                       }
+                       set_desc_id(ring[new_idx], new_idx);
+               }
+
+               /* hw link new descriptors */
+               for (i = curr_size-1; i < new_size; i++) {
+                       u16 new_idx = (ioat->tail+i) & (new_size-1);
+                       struct ioat_ring_ent *next = ring[(new_idx+1) & (new_size-1)];
+                       struct ioat_dma_descriptor *hw = ring[new_idx]->hw;
+
+                       hw->next = next->txd.phys;
+               }
+       } else {
+               struct ioat_dma_descriptor *hw;
+               struct ioat_ring_ent *next;
+
+               /* copy current descriptors to the new ring, dropping the
+                * removed descriptors
+                */
+               for (i = 0; i < new_size; i++) {
+                       u16 curr_idx = (ioat->tail+i) & (curr_size-1);
+                       u16 new_idx = (ioat->tail+i) & (new_size-1);
+
+                       ring[new_idx] = ioat->ring[curr_idx];
+                       set_desc_id(ring[new_idx], new_idx);
+               }
+
+               /* free deleted descriptors */
+               for (i = new_size; i < curr_size; i++) {
+                       struct ioat_ring_ent *ent;
+
+                       ent = ioat2_get_ring_ent(ioat, ioat->tail+i);
+                       ioat2_free_ring_ent(ent, c);
+               }
+
+               /* fix up hardware ring */
+               hw = ring[(ioat->tail+new_size-1) & (new_size-1)]->hw;
+               next = ring[(ioat->tail+new_size) & (new_size-1)];
+               hw->next = next->txd.phys;
+       }
+
+       dev_dbg(to_dev(chan), "%s: allocated %d descriptors\n",
+               __func__, new_size);
+
+       kfree(ioat->ring);
+       ioat->ring = ring;
+       ioat->alloc_order = order;
+
+       return true;
+}
+
+/**
+ * ioat2_alloc_and_lock - common descriptor alloc boilerplate for ioat2,3 ops
+ * @idx: gets starting descriptor index on successful allocation
+ * @ioat: ioat2,3 channel (ring) to operate on
+ * @num_descs: allocation length
+ */
+int ioat2_alloc_and_lock(u16 *idx, struct ioat2_dma_chan *ioat, int num_descs)
+{
+       struct ioat_chan_common *chan = &ioat->base;
+
+       spin_lock_bh(&ioat->ring_lock);
+       /* never allow the last descriptor to be consumed, we need at
+        * least one free at all times to allow for on-the-fly ring
+        * resizing.
+        */
+       while (unlikely(ioat2_ring_space(ioat) <= num_descs)) {
+               if (reshape_ring(ioat, ioat->alloc_order + 1) &&
+                   ioat2_ring_space(ioat) > num_descs)
+                               break;
+
+               if (printk_ratelimit())
+                       dev_dbg(to_dev(chan),
+                               "%s: ring full! num_descs: %d (%x:%x:%x)\n",
+                               __func__, num_descs, ioat->head, ioat->tail,
+                               ioat->issued);
+               spin_unlock_bh(&ioat->ring_lock);
+
+               /* progress reclaim in the allocation failure case we
+                * may be called under bh_disabled so we need to trigger
+                * the timer event directly
+                */
+               spin_lock_bh(&chan->cleanup_lock);
+               if (jiffies > chan->timer.expires &&
+                   timer_pending(&chan->timer)) {
+                       struct ioatdma_device *device = chan->device;
+
+                       mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+                       spin_unlock_bh(&chan->cleanup_lock);
+                       device->timer_fn((unsigned long) ioat);
+               } else
+                       spin_unlock_bh(&chan->cleanup_lock);
+               return -ENOMEM;
+       }
+
+       dev_dbg(to_dev(chan), "%s: num_descs: %d (%x:%x:%x)\n",
+               __func__, num_descs, ioat->head, ioat->tail, ioat->issued);
+
+       *idx = ioat2_desc_alloc(ioat, num_descs);
+       return 0;  /* with ioat->ring_lock held */
+}
+
+struct dma_async_tx_descriptor *
+ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
+                          dma_addr_t dma_src, size_t len, unsigned long flags)
+{
+       struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+       struct ioat_dma_descriptor *hw;
+       struct ioat_ring_ent *desc;
+       dma_addr_t dst = dma_dest;
+       dma_addr_t src = dma_src;
+       size_t total_len = len;
+       int num_descs;
+       u16 idx;
+       int i;
+
+       num_descs = ioat2_xferlen_to_descs(ioat, len);
+       if (likely(num_descs) &&
+           ioat2_alloc_and_lock(&idx, ioat, num_descs) == 0)
+               /* pass */;
+       else
+               return NULL;
+       i = 0;
+       do {
+               size_t copy = min_t(size_t, len, 1 << ioat->xfercap_log);
+
+               desc = ioat2_get_ring_ent(ioat, idx + i);
+               hw = desc->hw;
+
+               hw->size = copy;
+               hw->ctl = 0;
+               hw->src_addr = src;
+               hw->dst_addr = dst;
+
+               len -= copy;
+               dst += copy;
+               src += copy;
+               dump_desc_dbg(ioat, desc);
+       } while (++i < num_descs);
+
+       desc->txd.flags = flags;
+       desc->len = total_len;
+       hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
+       hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
+       hw->ctl_f.compl_write = 1;
+       dump_desc_dbg(ioat, desc);
+       /* we leave the channel locked to ensure in order submission */
+
+       return &desc->txd;
+}
+
+/**
+ * ioat2_free_chan_resources - release all the descriptors
+ * @chan: the channel to be cleaned
+ */
+void ioat2_free_chan_resources(struct dma_chan *c)
+{
+       struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+       struct ioat_chan_common *chan = &ioat->base;
+       struct ioatdma_device *device = chan->device;
+       struct ioat_ring_ent *desc;
+       const u16 total_descs = 1 << ioat->alloc_order;
+       int descs;
+       int i;
+
+       /* Before freeing channel resources first check
+        * if they have been previously allocated for this channel.
+        */
+       if (!ioat->ring)
+               return;
+
+       tasklet_disable(&chan->cleanup_task);
+       del_timer_sync(&chan->timer);
+       device->cleanup_tasklet((unsigned long) ioat);
+
+       /* Delay 100ms after reset to allow internal DMA logic to quiesce
+        * before removing DMA descriptor resources.
+        */
+       writeb(IOAT_CHANCMD_RESET,
+              chan->reg_base + IOAT_CHANCMD_OFFSET(chan->device->version));
+       mdelay(100);
+
+       spin_lock_bh(&ioat->ring_lock);
+       descs = ioat2_ring_space(ioat);
+       dev_dbg(to_dev(chan), "freeing %d idle descriptors\n", descs);
+       for (i = 0; i < descs; i++) {
+               desc = ioat2_get_ring_ent(ioat, ioat->head + i);
+               ioat2_free_ring_ent(desc, c);
+       }
+
+       if (descs < total_descs)
+               dev_err(to_dev(chan), "Freeing %d in use descriptors!\n",
+                       total_descs - descs);
+
+       for (i = 0; i < total_descs - descs; i++) {
+               desc = ioat2_get_ring_ent(ioat, ioat->tail + i);
+               dump_desc_dbg(ioat, desc);
+               ioat2_free_ring_ent(desc, c);
+       }
+
+       kfree(ioat->ring);
+       ioat->ring = NULL;
+       ioat->alloc_order = 0;
+       pci_pool_free(device->completion_pool, chan->completion,
+                     chan->completion_dma);
+       spin_unlock_bh(&ioat->ring_lock);
+
+       chan->last_completion = 0;
+       chan->completion_dma = 0;
+       ioat->pending = 0;
+       ioat->dmacount = 0;
+}
+
+enum dma_status
+ioat2_is_complete(struct dma_chan *c, dma_cookie_t cookie,
+                    dma_cookie_t *done, dma_cookie_t *used)
+{
+       struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+       struct ioatdma_device *device = ioat->base.device;
+
+       if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS)
+               return DMA_SUCCESS;
+
+       device->cleanup_tasklet((unsigned long) ioat);
+
+       return ioat_is_complete(c, cookie, done, used);
+}
+
+static ssize_t ring_size_show(struct dma_chan *c, char *page)
+{
+       struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+
+       return sprintf(page, "%d\n", (1 << ioat->alloc_order) & ~1);
+}
+static struct ioat_sysfs_entry ring_size_attr = __ATTR_RO(ring_size);
+
+static ssize_t ring_active_show(struct dma_chan *c, char *page)
+{
+       struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+
+       /* ...taken outside the lock, no need to be precise */
+       return sprintf(page, "%d\n", ioat2_ring_active(ioat));
+}
+static struct ioat_sysfs_entry ring_active_attr = __ATTR_RO(ring_active);
+
+static struct attribute *ioat2_attrs[] = {
+       &ring_size_attr.attr,
+       &ring_active_attr.attr,
+       &ioat_cap_attr.attr,
+       &ioat_version_attr.attr,
+       NULL,
+};
+
+struct kobj_type ioat2_ktype = {
+       .sysfs_ops = &ioat_sysfs_ops,
+       .default_attrs = ioat2_attrs,
+};
+
+int __devinit ioat2_dma_probe(struct ioatdma_device *device, int dca)
+{
+       struct pci_dev *pdev = device->pdev;
+       struct dma_device *dma;
+       struct dma_chan *c;
+       struct ioat_chan_common *chan;
+       int err;
+
+       device->enumerate_channels = ioat2_enumerate_channels;
+       device->cleanup_tasklet = ioat2_cleanup_tasklet;
+       device->timer_fn = ioat2_timer_event;
+       device->self_test = ioat_dma_self_test;
+       dma = &device->common;
+       dma->device_prep_dma_memcpy = ioat2_dma_prep_memcpy_lock;
+       dma->device_issue_pending = ioat2_issue_pending;
+       dma->device_alloc_chan_resources = ioat2_alloc_chan_resources;
+       dma->device_free_chan_resources = ioat2_free_chan_resources;
+       dma->device_is_tx_complete = ioat2_is_complete;
+
+       err = ioat_probe(device);
+       if (err)
+               return err;
+       ioat_set_tcp_copy_break(2048);
+
+       list_for_each_entry(c, &dma->channels, device_node) {
+               chan = to_chan_common(c);
+               writel(IOAT_DCACTRL_CMPL_WRITE_ENABLE | IOAT_DMA_DCA_ANY_CPU,
+                      chan->reg_base + IOAT_DCACTRL_OFFSET);
+       }
+
+       err = ioat_register(device);
+       if (err)
+               return err;
+
+       ioat_kobject_add(device, &ioat2_ktype);
+
+       if (dca)
+               device->dca = ioat2_dca_init(pdev, device->reg_base);
+
+       return err;
+}
diff --git a/drivers/dma/ioat/dma_v2.h b/drivers/dma/ioat/dma_v2.h
new file mode 100644 (file)
index 0000000..1d849ef
--- /dev/null
@@ -0,0 +1,190 @@
+/*
+ * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59
+ * Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called COPYING.
+ */
+#ifndef IOATDMA_V2_H
+#define IOATDMA_V2_H
+
+#include <linux/dmaengine.h>
+#include "dma.h"
+#include "hw.h"
+
+
+extern int ioat_pending_level;
+extern int ioat_ring_alloc_order;
+
+/*
+ * workaround for IOAT ver.3.0 null descriptor issue
+ * (channel returns error when size is 0)
+ */
+#define NULL_DESC_BUFFER_SIZE 1
+
+#define IOAT_MAX_ORDER 16
+#define ioat_get_alloc_order() \
+       (min(ioat_ring_alloc_order, IOAT_MAX_ORDER))
+#define ioat_get_max_alloc_order() \
+       (min(ioat_ring_max_alloc_order, IOAT_MAX_ORDER))
+
+/* struct ioat2_dma_chan - ioat v2 / v3 channel attributes
+ * @base: common ioat channel parameters
+ * @xfercap_log; log2 of channel max transfer length (for fast division)
+ * @head: allocated index
+ * @issued: hardware notification point
+ * @tail: cleanup index
+ * @pending: lock free indicator for issued != head
+ * @dmacount: identical to 'head' except for occasionally resetting to zero
+ * @alloc_order: log2 of the number of allocated descriptors
+ * @ring: software ring buffer implementation of hardware ring
+ * @ring_lock: protects ring attributes
+ */
+struct ioat2_dma_chan {
+       struct ioat_chan_common base;
+       size_t xfercap_log;
+       u16 head;
+       u16 issued;
+       u16 tail;
+       u16 dmacount;
+       u16 alloc_order;
+       int pending;
+       struct ioat_ring_ent **ring;
+       spinlock_t ring_lock;
+};
+
+static inline struct ioat2_dma_chan *to_ioat2_chan(struct dma_chan *c)
+{
+       struct ioat_chan_common *chan = to_chan_common(c);
+
+       return container_of(chan, struct ioat2_dma_chan, base);
+}
+
+static inline u16 ioat2_ring_mask(struct ioat2_dma_chan *ioat)
+{
+       return (1 << ioat->alloc_order) - 1;
+}
+
+/* count of descriptors in flight with the engine */
+static inline u16 ioat2_ring_active(struct ioat2_dma_chan *ioat)
+{
+       return (ioat->head - ioat->tail) & ioat2_ring_mask(ioat);
+}
+
+/* count of descriptors pending submission to hardware */
+static inline u16 ioat2_ring_pending(struct ioat2_dma_chan *ioat)
+{
+       return (ioat->head - ioat->issued) & ioat2_ring_mask(ioat);
+}
+
+static inline u16 ioat2_ring_space(struct ioat2_dma_chan *ioat)
+{
+       u16 num_descs = ioat2_ring_mask(ioat) + 1;
+       u16 active = ioat2_ring_active(ioat);
+
+       BUG_ON(active > num_descs);
+
+       return num_descs - active;
+}
+
+/* assumes caller already checked space */
+static inline u16 ioat2_desc_alloc(struct ioat2_dma_chan *ioat, u16 len)
+{
+       ioat->head += len;
+       return ioat->head - len;
+}
+
+static inline u16 ioat2_xferlen_to_descs(struct ioat2_dma_chan *ioat, size_t len)
+{
+       u16 num_descs = len >> ioat->xfercap_log;
+
+       num_descs += !!(len & ((1 << ioat->xfercap_log) - 1));
+       return num_descs;
+}
+
+/**
+ * struct ioat_ring_ent - wrapper around hardware descriptor
+ * @hw: hardware DMA descriptor (for memcpy)
+ * @fill: hardware fill descriptor
+ * @xor: hardware xor descriptor
+ * @xor_ex: hardware xor extension descriptor
+ * @pq: hardware pq descriptor
+ * @pq_ex: hardware pq extension descriptor
+ * @pqu: hardware pq update descriptor
+ * @raw: hardware raw (un-typed) descriptor
+ * @txd: the generic software descriptor for all engines
+ * @len: total transaction length for unmap
+ * @result: asynchronous result of validate operations
+ * @id: identifier for debug
+ */
+
+struct ioat_ring_ent {
+       union {
+               struct ioat_dma_descriptor *hw;
+               struct ioat_fill_descriptor *fill;
+               struct ioat_xor_descriptor *xor;
+               struct ioat_xor_ext_descriptor *xor_ex;
+               struct ioat_pq_descriptor *pq;
+               struct ioat_pq_ext_descriptor *pq_ex;
+               struct ioat_pq_update_descriptor *pqu;
+               struct ioat_raw_descriptor *raw;
+       };
+       size_t len;
+       struct dma_async_tx_descriptor txd;
+       enum sum_check_flags *result;
+       #ifdef DEBUG
+       int id;
+       #endif
+};
+
+static inline struct ioat_ring_ent *
+ioat2_get_ring_ent(struct ioat2_dma_chan *ioat, u16 idx)
+{
+       return ioat->ring[idx & ioat2_ring_mask(ioat)];
+}
+
+static inline void ioat2_set_chainaddr(struct ioat2_dma_chan *ioat, u64 addr)
+{
+       struct ioat_chan_common *chan = &ioat->base;
+
+       writel(addr & 0x00000000FFFFFFFF,
+              chan->reg_base + IOAT2_CHAINADDR_OFFSET_LOW);
+       writel(addr >> 32,
+              chan->reg_base + IOAT2_CHAINADDR_OFFSET_HIGH);
+}
+
+int __devinit ioat2_dma_probe(struct ioatdma_device *dev, int dca);
+int __devinit ioat3_dma_probe(struct ioatdma_device *dev, int dca);
+struct dca_provider * __devinit ioat2_dca_init(struct pci_dev *pdev, void __iomem *iobase);
+struct dca_provider * __devinit ioat3_dca_init(struct pci_dev *pdev, void __iomem *iobase);
+int ioat2_alloc_and_lock(u16 *idx, struct ioat2_dma_chan *ioat, int num_descs);
+int ioat2_enumerate_channels(struct ioatdma_device *device);
+struct dma_async_tx_descriptor *
+ioat2_dma_prep_memcpy_lock(struct dma_chan *c, dma_addr_t dma_dest,
+                          dma_addr_t dma_src, size_t len, unsigned long flags);
+void ioat2_issue_pending(struct dma_chan *chan);
+int ioat2_alloc_chan_resources(struct dma_chan *c);
+void ioat2_free_chan_resources(struct dma_chan *c);
+enum dma_status ioat2_is_complete(struct dma_chan *c, dma_cookie_t cookie,
+                                 dma_cookie_t *done, dma_cookie_t *used);
+void __ioat2_restart_chan(struct ioat2_dma_chan *ioat);
+bool reshape_ring(struct ioat2_dma_chan *ioat, int order);
+void __ioat2_issue_pending(struct ioat2_dma_chan *ioat);
+void ioat2_cleanup_tasklet(unsigned long data);
+void ioat2_timer_event(unsigned long data);
+extern struct kobj_type ioat2_ktype;
+extern struct kmem_cache *ioat2_cache;
+#endif /* IOATDMA_V2_H */
diff --git a/drivers/dma/ioat/dma_v3.c b/drivers/dma/ioat/dma_v3.c
new file mode 100644 (file)
index 0000000..35d1e33
--- /dev/null
@@ -0,0 +1,1223 @@
+/*
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2004 - 2009 Intel Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2004-2009 Intel Corporation. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   * Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in
+ *     the documentation and/or other materials provided with the
+ *     distribution.
+ *   * Neither the name of Intel Corporation nor the names of its
+ *     contributors may be used to endorse or promote products derived
+ *     from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Support routines for v3+ hardware
+ */
+
+#include <linux/pci.h>
+#include <linux/dmaengine.h>
+#include <linux/dma-mapping.h>
+#include "registers.h"
+#include "hw.h"
+#include "dma.h"
+#include "dma_v2.h"
+
+/* ioat hardware assumes at least two sources for raid operations */
+#define src_cnt_to_sw(x) ((x) + 2)
+#define src_cnt_to_hw(x) ((x) - 2)
+
+/* provide a lookup table for setting the source address in the base or
+ * extended descriptor of an xor or pq descriptor
+ */
+static const u8 xor_idx_to_desc __read_mostly = 0xd0;
+static const u8 xor_idx_to_field[] __read_mostly = { 1, 4, 5, 6, 7, 0, 1, 2 };
+static const u8 pq_idx_to_desc __read_mostly = 0xf8;
+static const u8 pq_idx_to_field[] __read_mostly = { 1, 4, 5, 0, 1, 2, 4, 5 };
+
+static dma_addr_t xor_get_src(struct ioat_raw_descriptor *descs[2], int idx)
+{
+       struct ioat_raw_descriptor *raw = descs[xor_idx_to_desc >> idx & 1];
+
+       return raw->field[xor_idx_to_field[idx]];
+}
+
+static void xor_set_src(struct ioat_raw_descriptor *descs[2],
+                       dma_addr_t addr, u32 offset, int idx)
+{
+       struct ioat_raw_descriptor *raw = descs[xor_idx_to_desc >> idx & 1];
+
+       raw->field[xor_idx_to_field[idx]] = addr + offset;
+}
+
+static dma_addr_t pq_get_src(struct ioat_raw_descriptor *descs[2], int idx)
+{
+       struct ioat_raw_descriptor *raw = descs[pq_idx_to_desc >> idx & 1];
+
+       return raw->field[pq_idx_to_field[idx]];
+}
+
+static void pq_set_src(struct ioat_raw_descriptor *descs[2],
+                      dma_addr_t addr, u32 offset, u8 coef, int idx)
+{
+       struct ioat_pq_descriptor *pq = (struct ioat_pq_descriptor *) descs[0];
+       struct ioat_raw_descriptor *raw = descs[pq_idx_to_desc >> idx & 1];
+
+       raw->field[pq_idx_to_field[idx]] = addr + offset;
+       pq->coef[idx] = coef;
+}
+
+static void ioat3_dma_unmap(struct ioat2_dma_chan *ioat,
+                           struct ioat_ring_ent *desc, int idx)
+{
+       struct ioat_chan_common *chan = &ioat->base;
+       struct pci_dev *pdev = chan->device->pdev;
+       size_t len = desc->len;
+       size_t offset = len - desc->hw->size;
+       struct dma_async_tx_descriptor *tx = &desc->txd;
+       enum dma_ctrl_flags flags = tx->flags;
+
+       switch (desc->hw->ctl_f.op) {
+       case IOAT_OP_COPY:
+               if (!desc->hw->ctl_f.null) /* skip 'interrupt' ops */
+                       ioat_dma_unmap(chan, flags, len, desc->hw);
+               break;
+       case IOAT_OP_FILL: {
+               struct ioat_fill_descriptor *hw = desc->fill;
+
+               if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP))
+                       ioat_unmap(pdev, hw->dst_addr - offset, len,
+                                  PCI_DMA_FROMDEVICE, flags, 1);
+               break;
+       }
+       case IOAT_OP_XOR_VAL:
+       case IOAT_OP_XOR: {
+               struct ioat_xor_descriptor *xor = desc->xor;
+               struct ioat_ring_ent *ext;
+               struct ioat_xor_ext_descriptor *xor_ex = NULL;
+               int src_cnt = src_cnt_to_sw(xor->ctl_f.src_cnt);
+               struct ioat_raw_descriptor *descs[2];
+               int i;
+
+               if (src_cnt > 5) {
+                       ext = ioat2_get_ring_ent(ioat, idx + 1);
+                       xor_ex = ext->xor_ex;
+               }
+
+               if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
+                       descs[0] = (struct ioat_raw_descriptor *) xor;
+                       descs[1] = (struct ioat_raw_descriptor *) xor_ex;
+                       for (i = 0; i < src_cnt; i++) {
+                               dma_addr_t src = xor_get_src(descs, i);
+
+                               ioat_unmap(pdev, src - offset, len,
+                                          PCI_DMA_TODEVICE, flags, 0);
+                       }
+
+                       /* dest is a source in xor validate operations */
+                       if (xor->ctl_f.op == IOAT_OP_XOR_VAL) {
+                               ioat_unmap(pdev, xor->dst_addr - offset, len,
+                                          PCI_DMA_TODEVICE, flags, 1);
+                               break;
+                       }
+               }
+
+               if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP))
+                       ioat_unmap(pdev, xor->dst_addr - offset, len,
+                                  PCI_DMA_FROMDEVICE, flags, 1);
+               break;
+       }
+       case IOAT_OP_PQ_VAL:
+       case IOAT_OP_PQ: {
+               struct ioat_pq_descriptor *pq = desc->pq;
+               struct ioat_ring_ent *ext;
+               struct ioat_pq_ext_descriptor *pq_ex = NULL;
+               int src_cnt = src_cnt_to_sw(pq->ctl_f.src_cnt);
+               struct ioat_raw_descriptor *descs[2];
+               int i;
+
+               if (src_cnt > 3) {
+                       ext = ioat2_get_ring_ent(ioat, idx + 1);
+                       pq_ex = ext->pq_ex;
+               }
+
+               /* in the 'continue' case don't unmap the dests as sources */
+               if (dmaf_p_disabled_continue(flags))
+                       src_cnt--;
+               else if (dmaf_continue(flags))
+                       src_cnt -= 3;
+
+               if (!(flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
+                       descs[0] = (struct ioat_raw_descriptor *) pq;
+                       descs[1] = (struct ioat_raw_descriptor *) pq_ex;
+                       for (i = 0; i < src_cnt; i++) {
+                               dma_addr_t src = pq_get_src(descs, i);
+
+                               ioat_unmap(pdev, src - offset, len,
+                                          PCI_DMA_TODEVICE, flags, 0);
+                       }
+
+                       /* the dests are sources in pq validate operations */
+                       if (pq->ctl_f.op == IOAT_OP_XOR_VAL) {
+                               if (!(flags & DMA_PREP_PQ_DISABLE_P))
+                                       ioat_unmap(pdev, pq->p_addr - offset,
+                                                  len, PCI_DMA_TODEVICE, flags, 0);
+                               if (!(flags & DMA_PREP_PQ_DISABLE_Q))
+                                       ioat_unmap(pdev, pq->q_addr - offset,
+                                                  len, PCI_DMA_TODEVICE, flags, 0);
+                               break;
+                       }
+               }
+
+               if (!(flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
+                       if (!(flags & DMA_PREP_PQ_DISABLE_P))
+                               ioat_unmap(pdev, pq->p_addr - offset, len,
+                                          PCI_DMA_BIDIRECTIONAL, flags, 1);
+                       if (!(flags & DMA_PREP_PQ_DISABLE_Q))
+                               ioat_unmap(pdev, pq->q_addr - offset, len,
+                                          PCI_DMA_BIDIRECTIONAL, flags, 1);
+               }
+               break;
+       }
+       default:
+               dev_err(&pdev->dev, "%s: unknown op type: %#x\n",
+                       __func__, desc->hw->ctl_f.op);
+       }
+}
+
+static bool desc_has_ext(struct ioat_ring_ent *desc)
+{
+       struct ioat_dma_descriptor *hw = desc->hw;
+
+       if (hw->ctl_f.op == IOAT_OP_XOR ||
+           hw->ctl_f.op == IOAT_OP_XOR_VAL) {
+               struct ioat_xor_descriptor *xor = desc->xor;
+
+               if (src_cnt_to_sw(xor->ctl_f.src_cnt) > 5)
+                       return true;
+       } else if (hw->ctl_f.op == IOAT_OP_PQ ||
+                  hw->ctl_f.op == IOAT_OP_PQ_VAL) {
+               struct ioat_pq_descriptor *pq = desc->pq;
+
+               if (src_cnt_to_sw(pq->ctl_f.src_cnt) > 3)
+                       return true;
+       }
+
+       return false;
+}
+
+/**
+ * __cleanup - reclaim used descriptors
+ * @ioat: channel (ring) to clean
+ *
+ * The difference from the dma_v2.c __cleanup() is that this routine
+ * handles extended descriptors and dma-unmapping raid operations.
+ */
+static void __cleanup(struct ioat2_dma_chan *ioat, unsigned long phys_complete)
+{
+       struct ioat_chan_common *chan = &ioat->base;
+       struct ioat_ring_ent *desc;
+       bool seen_current = false;
+       u16 active;
+       int i;
+
+       dev_dbg(to_dev(chan), "%s: head: %#x tail: %#x issued: %#x\n",
+               __func__, ioat->head, ioat->tail, ioat->issued);
+
+       active = ioat2_ring_active(ioat);
+       for (i = 0; i < active && !seen_current; i++) {
+               struct dma_async_tx_descriptor *tx;
+
+               prefetch(ioat2_get_ring_ent(ioat, ioat->tail + i + 1));
+               desc = ioat2_get_ring_ent(ioat, ioat->tail + i);
+               dump_desc_dbg(ioat, desc);
+               tx = &desc->txd;
+               if (tx->cookie) {
+                       chan->completed_cookie = tx->cookie;
+                       ioat3_dma_unmap(ioat, desc, ioat->tail + i);
+                       tx->cookie = 0;
+                       if (tx->callback) {
+                               tx->callback(tx->callback_param);
+                               tx->callback = NULL;
+                       }
+               }
+
+               if (tx->phys == phys_complete)
+                       seen_current = true;
+
+               /* skip extended descriptors */
+               if (desc_has_ext(desc)) {
+                       BUG_ON(i + 1 >= active);
+                       i++;
+               }
+       }
+       ioat->tail += i;
+       BUG_ON(!seen_current); /* no active descs have written a completion? */
+       chan->last_completion = phys_complete;
+       if (ioat->head == ioat->tail) {
+               dev_dbg(to_dev(chan), "%s: cancel completion timeout\n",
+                       __func__);
+               clear_bit(IOAT_COMPLETION_PENDING, &chan->state);
+               mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
+       }
+}
+
+static void ioat3_cleanup(struct ioat2_dma_chan *ioat)
+{
+       struct ioat_chan_common *chan = &ioat->base;
+       unsigned long phys_complete;
+
+       prefetch(chan->completion);
+
+       if (!spin_trylock_bh(&chan->cleanup_lock))
+               return;
+
+       if (!ioat_cleanup_preamble(chan, &phys_complete)) {
+               spin_unlock_bh(&chan->cleanup_lock);
+               return;
+       }
+
+       if (!spin_trylock_bh(&ioat->ring_lock)) {
+               spin_unlock_bh(&chan->cleanup_lock);
+               return;
+       }
+
+       __cleanup(ioat, phys_complete);
+
+       spin_unlock_bh(&ioat->ring_lock);
+       spin_unlock_bh(&chan->cleanup_lock);
+}
+
+static void ioat3_cleanup_tasklet(unsigned long data)
+{
+       struct ioat2_dma_chan *ioat = (void *) data;
+
+       ioat3_cleanup(ioat);
+       writew(IOAT_CHANCTRL_RUN | IOAT3_CHANCTRL_COMPL_DCA_EN,
+              ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);
+}
+
+static void ioat3_restart_channel(struct ioat2_dma_chan *ioat)
+{
+       struct ioat_chan_common *chan = &ioat->base;
+       unsigned long phys_complete;
+       u32 status;
+
+       status = ioat_chansts(chan);
+       if (is_ioat_active(status) || is_ioat_idle(status))
+               ioat_suspend(chan);
+       while (is_ioat_active(status) || is_ioat_idle(status)) {
+               status = ioat_chansts(chan);
+               cpu_relax();
+       }
+
+       if (ioat_cleanup_preamble(chan, &phys_complete))
+               __cleanup(ioat, phys_complete);
+
+       __ioat2_restart_chan(ioat);
+}
+
+static void ioat3_timer_event(unsigned long data)
+{
+       struct ioat2_dma_chan *ioat = (void *) data;
+       struct ioat_chan_common *chan = &ioat->base;
+
+       spin_lock_bh(&chan->cleanup_lock);
+       if (test_bit(IOAT_COMPLETION_PENDING, &chan->state)) {
+               unsigned long phys_complete;
+               u64 status;
+
+               spin_lock_bh(&ioat->ring_lock);
+               status = ioat_chansts(chan);
+
+               /* when halted due to errors check for channel
+                * programming errors before advancing the completion state
+                */
+               if (is_ioat_halted(status)) {
+                       u32 chanerr;
+
+                       chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
+                       BUG_ON(is_ioat_bug(chanerr));
+               }
+
+               /* if we haven't made progress and we have already
+                * acknowledged a pending completion once, then be more
+                * forceful with a restart
+                */
+               if (ioat_cleanup_preamble(chan, &phys_complete))
+                       __cleanup(ioat, phys_complete);
+               else if (test_bit(IOAT_COMPLETION_ACK, &chan->state))
+                       ioat3_restart_channel(ioat);
+               else {
+                       set_bit(IOAT_COMPLETION_ACK, &chan->state);
+                       mod_timer(&chan->timer, jiffies + COMPLETION_TIMEOUT);
+               }
+               spin_unlock_bh(&ioat->ring_lock);
+       } else {
+               u16 active;
+
+               /* if the ring is idle, empty, and oversized try to step
+                * down the size
+                */
+               spin_lock_bh(&ioat->ring_lock);
+               active = ioat2_ring_active(ioat);
+               if (active == 0 && ioat->alloc_order > ioat_get_alloc_order())
+                       reshape_ring(ioat, ioat->alloc_order-1);
+               spin_unlock_bh(&ioat->ring_lock);
+
+               /* keep shrinking until we get back to our minimum
+                * default size
+                */
+               if (ioat->alloc_order > ioat_get_alloc_order())
+                       mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
+       }
+       spin_unlock_bh(&chan->cleanup_lock);
+}
+
+static enum dma_status
+ioat3_is_complete(struct dma_chan *c, dma_cookie_t cookie,
+                 dma_cookie_t *done, dma_cookie_t *used)
+{
+       struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+
+       if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS)
+               return DMA_SUCCESS;
+
+       ioat3_cleanup(ioat);
+
+       return ioat_is_complete(c, cookie, done, used);
+}
+
+static struct dma_async_tx_descriptor *
+ioat3_prep_memset_lock(struct dma_chan *c, dma_addr_t dest, int value,
+                      size_t len, unsigned long flags)
+{
+       struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+       struct ioat_ring_ent *desc;
+       size_t total_len = len;
+       struct ioat_fill_descriptor *fill;
+       int num_descs;
+       u64 src_data = (0x0101010101010101ULL) * (value & 0xff);
+       u16 idx;
+       int i;
+
+       num_descs = ioat2_xferlen_to_descs(ioat, len);
+       if (likely(num_descs) &&
+           ioat2_alloc_and_lock(&idx, ioat, num_descs) == 0)
+               /* pass */;
+       else
+               return NULL;
+       i = 0;
+       do {
+               size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log);
+
+               desc = ioat2_get_ring_ent(ioat, idx + i);
+               fill = desc->fill;
+
+               fill->size = xfer_size;
+               fill->src_data = src_data;
+               fill->dst_addr = dest;
+               fill->ctl = 0;
+               fill->ctl_f.op = IOAT_OP_FILL;
+
+               len -= xfer_size;
+               dest += xfer_size;
+               dump_desc_dbg(ioat, desc);
+       } while (++i < num_descs);
+
+       desc->txd.flags = flags;
+       desc->len = total_len;
+       fill->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
+       fill->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
+       fill->ctl_f.compl_write = 1;
+       dump_desc_dbg(ioat, desc);
+
+       /* we leave the channel locked to ensure in order submission */
+       return &desc->txd;
+}
+
+static struct dma_async_tx_descriptor *
+__ioat3_prep_xor_lock(struct dma_chan *c, enum sum_check_flags *result,
+                     dma_addr_t dest, dma_addr_t *src, unsigned int src_cnt,
+                     size_t len, unsigned long flags)
+{
+       struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+       struct ioat_ring_ent *compl_desc;
+       struct ioat_ring_ent *desc;
+       struct ioat_ring_ent *ext;
+       size_t total_len = len;
+       struct ioat_xor_descriptor *xor;
+       struct ioat_xor_ext_descriptor *xor_ex = NULL;
+       struct ioat_dma_descriptor *hw;
+       u32 offset = 0;
+       int num_descs;
+       int with_ext;
+       int i;
+       u16 idx;
+       u8 op = result ? IOAT_OP_XOR_VAL : IOAT_OP_XOR;
+
+       BUG_ON(src_cnt < 2);
+
+       num_descs = ioat2_xferlen_to_descs(ioat, len);
+       /* we need 2x the number of descriptors to cover greater than 5
+        * sources
+        */
+       if (src_cnt > 5) {
+               with_ext = 1;
+               num_descs *= 2;
+       } else
+               with_ext = 0;
+
+       /* completion writes from the raid engine may pass completion
+        * writes from the legacy engine, so we need one extra null
+        * (legacy) descriptor to ensure all completion writes arrive in
+        * order.
+        */
+       if (likely(num_descs) &&
+           ioat2_alloc_and_lock(&idx, ioat, num_descs+1) == 0)
+               /* pass */;
+       else
+               return NULL;
+       i = 0;
+       do {
+               struct ioat_raw_descriptor *descs[2];
+               size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log);
+               int s;
+
+               desc = ioat2_get_ring_ent(ioat, idx + i);
+               xor = desc->xor;
+
+               /* save a branch by unconditionally retrieving the
+                * extended descriptor xor_set_src() knows to not write
+                * to it in the single descriptor case
+                */
+               ext = ioat2_get_ring_ent(ioat, idx + i + 1);
+               xor_ex = ext->xor_ex;
+
+               descs[0] = (struct ioat_raw_descriptor *) xor;
+               descs[1] = (struct ioat_raw_descriptor *) xor_ex;
+               for (s = 0; s < src_cnt; s++)
+                       xor_set_src(descs, src[s], offset, s);
+               xor->size = xfer_size;
+               xor->dst_addr = dest + offset;
+               xor->ctl = 0;
+               xor->ctl_f.op = op;
+               xor->ctl_f.src_cnt = src_cnt_to_hw(src_cnt);
+
+               len -= xfer_size;
+               offset += xfer_size;
+               dump_desc_dbg(ioat, desc);
+       } while ((i += 1 + with_ext) < num_descs);
+
+       /* last xor descriptor carries the unmap parameters and fence bit */
+       desc->txd.flags = flags;
+       desc->len = total_len;
+       if (result)
+               desc->result = result;
+       xor->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
+
+       /* completion descriptor carries interrupt bit */
+       compl_desc = ioat2_get_ring_ent(ioat, idx + i);
+       compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT;
+       hw = compl_desc->hw;
+       hw->ctl = 0;
+       hw->ctl_f.null = 1;
+       hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
+       hw->ctl_f.compl_write = 1;
+       hw->size = NULL_DESC_BUFFER_SIZE;
+       dump_desc_dbg(ioat, compl_desc);
+
+       /* we leave the channel locked to ensure in order submission */
+       return &desc->txd;
+}
+
+static struct dma_async_tx_descriptor *
+ioat3_prep_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,
+              unsigned int src_cnt, size_t len, unsigned long flags)
+{
+       return __ioat3_prep_xor_lock(chan, NULL, dest, src, src_cnt, len, flags);
+}
+
+struct dma_async_tx_descriptor *
+ioat3_prep_xor_val(struct dma_chan *chan, dma_addr_t *src,
+                   unsigned int src_cnt, size_t len,
+                   enum sum_check_flags *result, unsigned long flags)
+{
+       /* the cleanup routine only sets bits on validate failure, it
+        * does not clear bits on validate success... so clear it here
+        */
+       *result = 0;
+
+       return __ioat3_prep_xor_lock(chan, result, src[0], &src[1],
+                                    src_cnt - 1, len, flags);
+}
+
+static void
+dump_pq_desc_dbg(struct ioat2_dma_chan *ioat, struct ioat_ring_ent *desc, struct ioat_ring_ent *ext)
+{
+       struct device *dev = to_dev(&ioat->base);
+       struct ioat_pq_descriptor *pq = desc->pq;
+       struct ioat_pq_ext_descriptor *pq_ex = ext ? ext->pq_ex : NULL;
+       struct ioat_raw_descriptor *descs[] = { (void *) pq, (void *) pq_ex };
+       int src_cnt = src_cnt_to_sw(pq->ctl_f.src_cnt);
+       int i;
+
+       dev_dbg(dev, "desc[%d]: (%#llx->%#llx) flags: %#x"
+               " sz: %#x ctl: %#x (op: %d int: %d compl: %d pq: '%s%s' src_cnt: %d)\n",
+               desc_id(desc), (unsigned long long) desc->txd.phys,
+               (unsigned long long) (pq_ex ? pq_ex->next : pq->next),
+               desc->txd.flags, pq->size, pq->ctl, pq->ctl_f.op, pq->ctl_f.int_en,
+               pq->ctl_f.compl_write,
+               pq->ctl_f.p_disable ? "" : "p", pq->ctl_f.q_disable ? "" : "q",
+               pq->ctl_f.src_cnt);
+       for (i = 0; i < src_cnt; i++)
+               dev_dbg(dev, "\tsrc[%d]: %#llx coef: %#x\n", i,
+                       (unsigned long long) pq_get_src(descs, i), pq->coef[i]);
+       dev_dbg(dev, "\tP: %#llx\n", pq->p_addr);
+       dev_dbg(dev, "\tQ: %#llx\n", pq->q_addr);
+}
+
+static struct dma_async_tx_descriptor *
+__ioat3_prep_pq_lock(struct dma_chan *c, enum sum_check_flags *result,
+                    const dma_addr_t *dst, const dma_addr_t *src,
+                    unsigned int src_cnt, const unsigned char *scf,
+                    size_t len, unsigned long flags)
+{
+       struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+       struct ioat_chan_common *chan = &ioat->base;
+       struct ioat_ring_ent *compl_desc;
+       struct ioat_ring_ent *desc;
+       struct ioat_ring_ent *ext;
+       size_t total_len = len;
+       struct ioat_pq_descriptor *pq;
+       struct ioat_pq_ext_descriptor *pq_ex = NULL;
+       struct ioat_dma_descriptor *hw;
+       u32 offset = 0;
+       int num_descs;
+       int with_ext;
+       int i, s;
+       u16 idx;
+       u8 op = result ? IOAT_OP_PQ_VAL : IOAT_OP_PQ;
+
+       dev_dbg(to_dev(chan), "%s\n", __func__);
+       /* the engine requires at least two sources (we provide
+        * at least 1 implied source in the DMA_PREP_CONTINUE case)
+        */
+       BUG_ON(src_cnt + dmaf_continue(flags) < 2);
+
+       num_descs = ioat2_xferlen_to_descs(ioat, len);
+       /* we need 2x the number of descriptors to cover greater than 3
+        * sources
+        */
+       if (src_cnt > 3 || flags & DMA_PREP_CONTINUE) {
+               with_ext = 1;
+               num_descs *= 2;
+       } else
+               with_ext = 0;
+
+       /* completion writes from the raid engine may pass completion
+        * writes from the legacy engine, so we need one extra null
+        * (legacy) descriptor to ensure all completion writes arrive in
+        * order.
+        */
+       if (likely(num_descs) &&
+           ioat2_alloc_and_lock(&idx, ioat, num_descs+1) == 0)
+               /* pass */;
+       else
+               return NULL;
+       i = 0;
+       do {
+               struct ioat_raw_descriptor *descs[2];
+               size_t xfer_size = min_t(size_t, len, 1 << ioat->xfercap_log);
+
+               desc = ioat2_get_ring_ent(ioat, idx + i);
+               pq = desc->pq;
+
+               /* save a branch by unconditionally retrieving the
+                * extended descriptor pq_set_src() knows to not write
+                * to it in the single descriptor case
+                */
+               ext = ioat2_get_ring_ent(ioat, idx + i + with_ext);
+               pq_ex = ext->pq_ex;
+
+               descs[0] = (struct ioat_raw_descriptor *) pq;
+               descs[1] = (struct ioat_raw_descriptor *) pq_ex;
+
+               for (s = 0; s < src_cnt; s++)
+                       pq_set_src(descs, src[s], offset, scf[s], s);
+
+               /* see the comment for dma_maxpq in include/linux/dmaengine.h */
+               if (dmaf_p_disabled_continue(flags))
+                       pq_set_src(descs, dst[1], offset, 1, s++);
+               else if (dmaf_continue(flags)) {
+                       pq_set_src(descs, dst[0], offset, 0, s++);
+                       pq_set_src(descs, dst[1], offset, 1, s++);
+                       pq_set_src(descs, dst[1], offset, 0, s++);
+               }
+               pq->size = xfer_size;
+               pq->p_addr = dst[0] + offset;
+               pq->q_addr = dst[1] + offset;
+               pq->ctl = 0;
+               pq->ctl_f.op = op;
+               pq->ctl_f.src_cnt = src_cnt_to_hw(s);
+               pq->ctl_f.p_disable = !!(flags & DMA_PREP_PQ_DISABLE_P);
+               pq->ctl_f.q_disable = !!(flags & DMA_PREP_PQ_DISABLE_Q);
+
+               len -= xfer_size;
+               offset += xfer_size;
+       } while ((i += 1 + with_ext) < num_descs);
+
+       /* last pq descriptor carries the unmap parameters and fence bit */
+       desc->txd.flags = flags;
+       desc->len = total_len;
+       if (result)
+               desc->result = result;
+       pq->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
+       dump_pq_desc_dbg(ioat, desc, ext);
+
+       /* completion descriptor carries interrupt bit */
+       compl_desc = ioat2_get_ring_ent(ioat, idx + i);
+       compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT;
+       hw = compl_desc->hw;
+       hw->ctl = 0;
+       hw->ctl_f.null = 1;
+       hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);
+       hw->ctl_f.compl_write = 1;
+       hw->size = NULL_DESC_BUFFER_SIZE;
+       dump_desc_dbg(ioat, compl_desc);
+
+       /* we leave the channel locked to ensure in order submission */
+       return &desc->txd;
+}
+
+static struct dma_async_tx_descriptor *
+ioat3_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,
+             unsigned int src_cnt, const unsigned char *scf, size_t len,
+             unsigned long flags)
+{
+       /* handle the single source multiply case from the raid6
+        * recovery path
+        */
+       if (unlikely((flags & DMA_PREP_PQ_DISABLE_P) && src_cnt == 1)) {
+               dma_addr_t single_source[2];
+               unsigned char single_source_coef[2];
+
+               BUG_ON(flags & DMA_PREP_PQ_DISABLE_Q);
+               single_source[0] = src[0];
+               single_source[1] = src[0];
+               single_source_coef[0] = scf[0];
+               single_source_coef[1] = 0;
+
+               return __ioat3_prep_pq_lock(chan, NULL, dst, single_source, 2,
+                                           single_source_coef, len, flags);
+       } else
+               return __ioat3_prep_pq_lock(chan, NULL, dst, src, src_cnt, scf,
+                                           len, flags);
+}
+
+struct dma_async_tx_descriptor *
+ioat3_prep_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,
+                 unsigned int src_cnt, const unsigned char *scf, size_t len,
+                 enum sum_check_flags *pqres, unsigned long flags)
+{
+       /* the cleanup routine only sets bits on validate failure, it
+        * does not clear bits on validate success... so clear it here
+        */
+       *pqres = 0;
+
+       return __ioat3_prep_pq_lock(chan, pqres, pq, src, src_cnt, scf, len,
+                                   flags);
+}
+
+static struct dma_async_tx_descriptor *
+ioat3_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src,
+                unsigned int src_cnt, size_t len, unsigned long flags)
+{
+       unsigned char scf[src_cnt];
+       dma_addr_t pq[2];
+
+       memset(scf, 0, src_cnt);
+       flags |= DMA_PREP_PQ_DISABLE_Q;
+       pq[0] = dst;
+       pq[1] = ~0;
+
+       return __ioat3_prep_pq_lock(chan, NULL, pq, src, src_cnt, scf, len,
+                                   flags);
+}
+
+struct dma_async_tx_descriptor *
+ioat3_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src,
+                    unsigned int src_cnt, size_t len,
+                    enum sum_check_flags *result, unsigned long flags)
+{
+       unsigned char scf[src_cnt];
+       dma_addr_t pq[2];
+
+       /* the cleanup routine only sets bits on validate failure, it
+        * does not clear bits on validate success... so clear it here
+        */
+       *result = 0;
+
+       memset(scf, 0, src_cnt);
+       flags |= DMA_PREP_PQ_DISABLE_Q;
+       pq[0] = src[0];
+       pq[1] = ~0;
+
+       return __ioat3_prep_pq_lock(chan, result, pq, &src[1], src_cnt - 1, scf,
+                                   len, flags);
+}
+
+static struct dma_async_tx_descriptor *
+ioat3_prep_interrupt_lock(struct dma_chan *c, unsigned long flags)
+{
+       struct ioat2_dma_chan *ioat = to_ioat2_chan(c);
+       struct ioat_ring_ent *desc;
+       struct ioat_dma_descriptor *hw;
+       u16 idx;
+
+       if (ioat2_alloc_and_lock(&idx, ioat, 1) == 0)
+               desc = ioat2_get_ring_ent(ioat, idx);
+       else
+               return NULL;
+
+       hw = desc->hw;
+       hw->ctl = 0;
+       hw->ctl_f.null = 1;
+       hw->ctl_f.int_en = 1;
+       hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE);
+       hw->ctl_f.compl_write = 1;
+       hw->size = NULL_DESC_BUFFER_SIZE;
+       hw->src_addr = 0;
+       hw->dst_addr = 0;
+
+       desc->txd.flags = flags;
+       desc->len = 1;
+
+       dump_desc_dbg(ioat, desc);
+
+       /* we leave the channel locked to ensure in order submission */
+       return &desc->txd;
+}
+
+static void __devinit ioat3_dma_test_callback(void *dma_async_param)
+{
+       struct completion *cmp = dma_async_param;
+
+       complete(cmp);
+}
+
+#define IOAT_NUM_SRC_TEST 6 /* must be <= 8 */
+static int __devinit ioat_xor_val_self_test(struct ioatdma_device *device)
+{
+       int i, src_idx;
+       struct page *dest;
+       struct page *xor_srcs[IOAT_NUM_SRC_TEST];
+       struct page *xor_val_srcs[IOAT_NUM_SRC_TEST + 1];
+       dma_addr_t dma_srcs[IOAT_NUM_SRC_TEST + 1];
+       dma_addr_t dma_addr, dest_dma;
+       struct dma_async_tx_descriptor *tx;
+       struct dma_chan *dma_chan;
+       dma_cookie_t cookie;
+       u8 cmp_byte = 0;
+       u32 cmp_word;
+       u32 xor_val_result;
+       int err = 0;
+       struct completion cmp;
+       unsigned long tmo;
+       struct device *dev = &device->pdev->dev;
+       struct dma_device *dma = &device->common;
+
+       dev_dbg(dev, "%s\n", __func__);
+
+       if (!dma_has_cap(DMA_XOR, dma->cap_mask))
+               return 0;
+
+       for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++) {
+               xor_srcs[src_idx] = alloc_page(GFP_KERNEL);
+               if (!xor_srcs[src_idx]) {
+                       while (src_idx--)
+                               __free_page(xor_srcs[src_idx]);
+                       return -ENOMEM;
+               }
+       }
+
+       dest = alloc_page(GFP_KERNEL);
+       if (!dest) {
+               while (src_idx--)
+                       __free_page(xor_srcs[src_idx]);
+               return -ENOMEM;
+       }
+
+       /* Fill in src buffers */
+       for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++) {
+               u8 *ptr = page_address(xor_srcs[src_idx]);
+               for (i = 0; i < PAGE_SIZE; i++)
+                       ptr[i] = (1 << src_idx);
+       }
+
+       for (src_idx = 0; src_idx < IOAT_NUM_SRC_TEST; src_idx++)
+               cmp_byte ^= (u8) (1 << src_idx);
+
+       cmp_word = (cmp_byte << 24) | (cmp_byte << 16) |
+                       (cmp_byte << 8) | cmp_byte;
+
+       memset(page_address(dest), 0, PAGE_SIZE);
+
+       dma_chan = container_of(dma->channels.next, struct dma_chan,
+                               device_node);
+       if (dma->device_alloc_chan_resources(dma_chan) < 1) {
+               err = -ENODEV;
+               goto out;
+       }
+
+       /* test xor */
+       dest_dma = dma_map_page(dev, dest, 0, PAGE_SIZE, DMA_FROM_DEVICE);
+       for (i = 0; i < IOAT_NUM_SRC_TEST; i++)
+               dma_srcs[i] = dma_map_page(dev, xor_srcs[i], 0, PAGE_SIZE,
+                                          DMA_TO_DEVICE);
+       tx = dma->device_prep_dma_xor(dma_chan, dest_dma, dma_srcs,
+                                     IOAT_NUM_SRC_TEST, PAGE_SIZE,
+                                     DMA_PREP_INTERRUPT);
+
+       if (!tx) {
+               dev_err(dev, "Self-test xor prep failed\n");
+               err = -ENODEV;
+               goto free_resources;
+       }
+
+       async_tx_ack(tx);
+       init_completion(&cmp);
+       tx->callback = ioat3_dma_test_callback;
+       tx->callback_param = &cmp;
+       cookie = tx->tx_submit(tx);
+       if (cookie < 0) {
+               dev_err(dev, "Self-test xor setup failed\n");
+               err = -ENODEV;
+               goto free_resources;
+       }
+       dma->device_issue_pending(dma_chan);
+
+       tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
+
+       if (dma->device_is_tx_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
+               dev_err(dev, "Self-test xor timed out\n");
+               err = -ENODEV;
+               goto free_resources;
+       }
+
+       dma_sync_single_for_cpu(dev, dest_dma, PAGE_SIZE, DMA_FROM_DEVICE);
+       for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) {
+               u32 *ptr = page_address(dest);
+               if (ptr[i] != cmp_word) {
+                       dev_err(dev, "Self-test xor failed compare\n");
+                       err = -ENODEV;
+                       goto free_resources;
+               }
+       }
+       dma_sync_single_for_device(dev, dest_dma, PAGE_SIZE, DMA_TO_DEVICE);
+
+       /* skip validate if the capability is not present */
+       if (!dma_has_cap(DMA_XOR_VAL, dma_chan->device->cap_mask))
+               goto free_resources;
+
+       /* validate the sources with the destintation page */
+       for (i = 0; i < IOAT_NUM_SRC_TEST; i++)
+               xor_val_srcs[i] = xor_srcs[i];
+       xor_val_srcs[i] = dest;
+
+       xor_val_result = 1;
+
+       for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++)
+               dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE,
+                                          DMA_TO_DEVICE);
+       tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs,
+                                         IOAT_NUM_SRC_TEST + 1, PAGE_SIZE,
+                                         &xor_val_result, DMA_PREP_INTERRUPT);
+       if (!tx) {
+               dev_err(dev, "Self-test zero prep failed\n");
+               err = -ENODEV;
+               goto free_resources;
+       }
+
+       async_tx_ack(tx);
+       init_completion(&cmp);
+       tx->callback = ioat3_dma_test_callback;
+       tx->callback_param = &cmp;
+       cookie = tx->tx_submit(tx);
+       if (cookie < 0) {
+               dev_err(dev, "Self-test zero setup failed\n");
+               err = -ENODEV;
+               goto free_resources;
+       }
+       dma->device_issue_pending(dma_chan);
+
+       tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
+
+       if (dma->device_is_tx_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
+               dev_err(dev, "Self-test validate timed out\n");
+               err = -ENODEV;
+               goto free_resources;
+       }
+
+       if (xor_val_result != 0) {
+               dev_err(dev, "Self-test validate failed compare\n");
+               err = -ENODEV;
+               goto free_resources;
+       }
+
+       /* skip memset if the capability is not present */
+       if (!dma_has_cap(DMA_MEMSET, dma_chan->device->cap_mask))
+               goto free_resources;
+
+       /* test memset */
+       dma_addr = dma_map_page(dev, dest, 0,
+                       PAGE_SIZE, DMA_FROM_DEVICE);
+       tx = dma->device_prep_dma_memset(dma_chan, dma_addr, 0, PAGE_SIZE,
+                                        DMA_PREP_INTERRUPT);
+       if (!tx) {
+               dev_err(dev, "Self-test memset prep failed\n");
+               err = -ENODEV;
+               goto free_resources;
+       }
+
+       async_tx_ack(tx);
+       init_completion(&cmp);
+       tx->callback = ioat3_dma_test_callback;
+       tx->callback_param = &cmp;
+       cookie = tx->tx_submit(tx);
+       if (cookie < 0) {
+               dev_err(dev, "Self-test memset setup failed\n");
+               err = -ENODEV;
+               goto free_resources;
+       }
+       dma->device_issue_pending(dma_chan);
+
+       tmo = wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000));
+
+       if (dma->device_is_tx_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
+               dev_err(dev, "Self-test memset timed out\n");
+               err = -ENODEV;
+               goto free_resources;
+       }
+
+       for (i = 0; i < PAGE_SIZE/sizeof(u32); i++) {
+               u32 *ptr = page_address(dest);
+               if (ptr[i]) {
+                       dev_err(dev, "Self-test memset failed compare\n");
+                       err = -ENODEV;
+                       goto free_resources;
+               }
+       }
+
+       /* test for non-zero parity sum */
+       xor_val_result = 0;
+       for (i = 0; i < IOAT_NUM_SRC_TEST + 1; i++)
+               dma_srcs[i] = dma_map_page(dev, xor_val_srcs[i], 0, PAGE_SIZE,
+                                          DMA_TO_DEVICE);
+       tx = dma->device_prep_dma_xor_val(dma_chan, dma_srcs,
+                                         IOAT_NUM_SRC_TEST + 1, PAGE_SIZE,
+                                         &xor_val_result, DMA_PREP_INTERRUPT);
+       if (!tx) {
+               dev_err(dev, "Self-test 2nd zero prep failed\n");
+               err = -ENODEV;
+               goto free_resources;
+       }
+
+       async_tx_ack(tx);
+       init_completion(&cmp);
+       tx->callback = ioat3_dma_test_callback;
+       tx->callback_param = &cmp;
+       cookie = tx->tx_submit(tx);
+       if (cookie < 0) {
+               dev_err(dev, "Self-test  2nd zero setup failed\n");
+               err = -ENODEV;
+               goto free_resources;
+       }
+       dma->device_issue_pending(dma_chan);
+
+       tmo = wait_for_completion_timeout(&cmp, msecs_to