*/
#include <linux/module.h>
+#include <linux/dma-mapping.h>
#include <asm/dma.h>
-#include <asm/sn/pcibr_provider.h>
+#include <asm/sn/intr.h>
#include <asm/sn/pcibus_provider_defs.h>
#include <asm/sn/pcidev.h>
#include <asm/sn/sn_sal.h>
-#define SG_ENT_VIRT_ADDRESS(sg) (page_address((sg)->page) + (sg)->offset)
+#define SG_ENT_VIRT_ADDRESS(sg) (sg_virt((sg)))
#define SG_ENT_PHYS_ADDRESS(SG) virt_to_phys(SG_ENT_VIRT_ADDRESS(SG))
/**
* this function. Of course, SN only supports devices that have 32 or more
* address bits when using the PMU.
*/
-int sn_dma_supported(struct device *dev, u64 mask)
+static int sn_dma_supported(struct device *dev, u64 mask)
{
BUG_ON(dev->bus != &pci_bus_type);
return 0;
return 1;
}
-EXPORT_SYMBOL(sn_dma_supported);
/**
* sn_dma_set_mask - set the DMA mask
* queue for a SCSI controller). See Documentation/DMA-API.txt for
* more information.
*/
-void *sn_dma_alloc_coherent(struct device *dev, size_t size,
- dma_addr_t * dma_handle, gfp_t flags)
+static void *sn_dma_alloc_coherent(struct device *dev, size_t size,
+ dma_addr_t * dma_handle, gfp_t flags)
{
void *cpuaddr;
unsigned long phys_addr;
*/
node = pcibus_to_node(pdev->bus);
if (likely(node >=0)) {
- struct page *p = alloc_pages_node(node, GFP_ATOMIC, get_order(size));
+ struct page *p = alloc_pages_exact_node(node,
+ flags, get_order(size));
if (likely(p))
cpuaddr = page_address(p);
else
return NULL;
} else
- cpuaddr = (void *)__get_free_pages(GFP_ATOMIC, get_order(size));
+ cpuaddr = (void *)__get_free_pages(flags, get_order(size));
if (unlikely(!cpuaddr))
return NULL;
* resources.
*/
- *dma_handle = provider->dma_map_consistent(pdev, phys_addr, size);
+ *dma_handle = provider->dma_map_consistent(pdev, phys_addr, size,
+ SN_DMA_ADDR_PHYS);
if (!*dma_handle) {
- printk(KERN_ERR "%s: out of ATEs\n", __FUNCTION__);
+ printk(KERN_ERR "%s: out of ATEs\n", __func__);
free_pages((unsigned long)cpuaddr, get_order(size));
return NULL;
}
return cpuaddr;
}
-EXPORT_SYMBOL(sn_dma_alloc_coherent);
/**
* sn_pci_free_coherent - free memory associated with coherent DMAable region
* Frees the memory allocated by dma_alloc_coherent(), potentially unmapping
* any associated IOMMU mappings.
*/
-void sn_dma_free_coherent(struct device *dev, size_t size, void *cpu_addr,
- dma_addr_t dma_handle)
+static void sn_dma_free_coherent(struct device *dev, size_t size, void *cpu_addr,
+ dma_addr_t dma_handle)
{
struct pci_dev *pdev = to_pci_dev(dev);
struct sn_pcibus_provider *provider = SN_PCIDEV_BUSPROVIDER(pdev);
provider->dma_unmap(pdev, dma_handle, 0);
free_pages((unsigned long)cpu_addr, get_order(size));
}
-EXPORT_SYMBOL(sn_dma_free_coherent);
/**
- * sn_dma_map_single - map a single page for DMA
+ * sn_dma_map_single_attrs - map a single page for DMA
* @dev: device to map for
* @cpu_addr: kernel virtual address of the region to map
* @size: size of the region
* @direction: DMA direction
+ * @attrs: optional dma attributes
*
* Map the region pointed to by @cpu_addr for DMA and return the
* DMA address.
* no way of saving the dmamap handle from the alloc to later free
* (which is pretty much unacceptable).
*
+ * mappings with the DMA_ATTR_WRITE_BARRIER get mapped with
+ * dma_map_consistent() so that writes force a flush of pending DMA.
+ * (See "SGI Altix Architecture Considerations for Linux Device Drivers",
+ * Document Number: 007-4763-001)
+ *
* TODO: simplify our interface;
* figure out how to save dmamap handle so can use two step.
*/
-dma_addr_t sn_dma_map_single(struct device *dev, void *cpu_addr, size_t size,
- int direction)
+static dma_addr_t sn_dma_map_page(struct device *dev, struct page *page,
+ unsigned long offset, size_t size,
+ enum dma_data_direction dir,
+ struct dma_attrs *attrs)
{
+ void *cpu_addr = page_address(page) + offset;
dma_addr_t dma_addr;
unsigned long phys_addr;
struct pci_dev *pdev = to_pci_dev(dev);
struct sn_pcibus_provider *provider = SN_PCIDEV_BUSPROVIDER(pdev);
+ int dmabarr;
+
+ dmabarr = dma_get_attr(DMA_ATTR_WRITE_BARRIER, attrs);
BUG_ON(dev->bus != &pci_bus_type);
phys_addr = __pa(cpu_addr);
- dma_addr = provider->dma_map(pdev, phys_addr, size);
+ if (dmabarr)
+ dma_addr = provider->dma_map_consistent(pdev, phys_addr,
+ size, SN_DMA_ADDR_PHYS);
+ else
+ dma_addr = provider->dma_map(pdev, phys_addr, size,
+ SN_DMA_ADDR_PHYS);
+
if (!dma_addr) {
- printk(KERN_ERR "%s: out of ATEs\n", __FUNCTION__);
+ printk(KERN_ERR "%s: out of ATEs\n", __func__);
return 0;
}
return dma_addr;
}
-EXPORT_SYMBOL(sn_dma_map_single);
/**
- * sn_dma_unmap_single - unamp a DMA mapped page
+ * sn_dma_unmap_single_attrs - unamp a DMA mapped page
* @dev: device to sync
* @dma_addr: DMA address to sync
* @size: size of region
* @direction: DMA direction
+ * @attrs: optional dma attributes
*
* This routine is supposed to sync the DMA region specified
* by @dma_handle into the coherence domain. On SN, we're always cache
* coherent, so we just need to free any ATEs associated with this mapping.
*/
-void sn_dma_unmap_single(struct device *dev, dma_addr_t dma_addr, size_t size,
- int direction)
+static void sn_dma_unmap_page(struct device *dev, dma_addr_t dma_addr,
+ size_t size, enum dma_data_direction dir,
+ struct dma_attrs *attrs)
{
struct pci_dev *pdev = to_pci_dev(dev);
struct sn_pcibus_provider *provider = SN_PCIDEV_BUSPROVIDER(pdev);
BUG_ON(dev->bus != &pci_bus_type);
- provider->dma_unmap(pdev, dma_addr, direction);
+ provider->dma_unmap(pdev, dma_addr, dir);
}
-EXPORT_SYMBOL(sn_dma_unmap_single);
/**
* sn_dma_unmap_sg - unmap a DMA scatterlist
* @sg: scatterlist to unmap
* @nhwentries: number of scatterlist entries
* @direction: DMA direction
+ * @attrs: optional dma attributes
*
* Unmap a set of streaming mode DMA translations.
*/
-void sn_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
- int nhwentries, int direction)
+static void sn_dma_unmap_sg(struct device *dev, struct scatterlist *sgl,
+ int nhwentries, enum dma_data_direction dir,
+ struct dma_attrs *attrs)
{
int i;
struct pci_dev *pdev = to_pci_dev(dev);
struct sn_pcibus_provider *provider = SN_PCIDEV_BUSPROVIDER(pdev);
+ struct scatterlist *sg;
BUG_ON(dev->bus != &pci_bus_type);
- for (i = 0; i < nhwentries; i++, sg++) {
- provider->dma_unmap(pdev, sg->dma_address, direction);
+ for_each_sg(sgl, sg, nhwentries, i) {
+ provider->dma_unmap(pdev, sg->dma_address, dir);
sg->dma_address = (dma_addr_t) NULL;
sg->dma_length = 0;
}
}
-EXPORT_SYMBOL(sn_dma_unmap_sg);
/**
* sn_dma_map_sg - map a scatterlist for DMA
* @sg: scatterlist to map
* @nhwentries: number of entries
* @direction: direction of the DMA transaction
+ * @attrs: optional dma attributes
+ *
+ * mappings with the DMA_ATTR_WRITE_BARRIER get mapped with
+ * dma_map_consistent() so that writes force a flush of pending DMA.
+ * (See "SGI Altix Architecture Considerations for Linux Device Drivers",
+ * Document Number: 007-4763-001)
*
* Maps each entry of @sg for DMA.
*/
-int sn_dma_map_sg(struct device *dev, struct scatterlist *sg, int nhwentries,
- int direction)
+static int sn_dma_map_sg(struct device *dev, struct scatterlist *sgl,
+ int nhwentries, enum dma_data_direction dir,
+ struct dma_attrs *attrs)
{
unsigned long phys_addr;
- struct scatterlist *saved_sg = sg;
+ struct scatterlist *saved_sg = sgl, *sg;
struct pci_dev *pdev = to_pci_dev(dev);
struct sn_pcibus_provider *provider = SN_PCIDEV_BUSPROVIDER(pdev);
int i;
+ int dmabarr;
+
+ dmabarr = dma_get_attr(DMA_ATTR_WRITE_BARRIER, attrs);
BUG_ON(dev->bus != &pci_bus_type);
/*
* Setup a DMA address for each entry in the scatterlist.
*/
- for (i = 0; i < nhwentries; i++, sg++) {
+ for_each_sg(sgl, sg, nhwentries, i) {
+ dma_addr_t dma_addr;
phys_addr = SG_ENT_PHYS_ADDRESS(sg);
- sg->dma_address = provider->dma_map(pdev,
- phys_addr, sg->length);
+ if (dmabarr)
+ dma_addr = provider->dma_map_consistent(pdev,
+ phys_addr,
+ sg->length,
+ SN_DMA_ADDR_PHYS);
+ else
+ dma_addr = provider->dma_map(pdev, phys_addr,
+ sg->length,
+ SN_DMA_ADDR_PHYS);
+ sg->dma_address = dma_addr;
if (!sg->dma_address) {
- printk(KERN_ERR "%s: out of ATEs\n", __FUNCTION__);
+ printk(KERN_ERR "%s: out of ATEs\n", __func__);
/*
* Free any successfully allocated entries.
*/
if (i > 0)
- sn_dma_unmap_sg(dev, saved_sg, i, direction);
+ sn_dma_unmap_sg(dev, saved_sg, i, dir, attrs);
return 0;
}
return nhwentries;
}
-EXPORT_SYMBOL(sn_dma_map_sg);
-void sn_dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle,
- size_t size, int direction)
+static void sn_dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle,
+ size_t size, enum dma_data_direction dir)
{
BUG_ON(dev->bus != &pci_bus_type);
}
-EXPORT_SYMBOL(sn_dma_sync_single_for_cpu);
-void sn_dma_sync_single_for_device(struct device *dev, dma_addr_t dma_handle,
- size_t size, int direction)
+static void sn_dma_sync_single_for_device(struct device *dev, dma_addr_t dma_handle,
+ size_t size,
+ enum dma_data_direction dir)
{
BUG_ON(dev->bus != &pci_bus_type);
}
-EXPORT_SYMBOL(sn_dma_sync_single_for_device);
-void sn_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
- int nelems, int direction)
+static void sn_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
+ int nelems, enum dma_data_direction dir)
{
BUG_ON(dev->bus != &pci_bus_type);
}
-EXPORT_SYMBOL(sn_dma_sync_sg_for_cpu);
-void sn_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
- int nelems, int direction)
+static void sn_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
+ int nelems, enum dma_data_direction dir)
{
BUG_ON(dev->bus != &pci_bus_type);
}
-EXPORT_SYMBOL(sn_dma_sync_sg_for_device);
-int sn_dma_mapping_error(dma_addr_t dma_addr)
+static int sn_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
{
return 0;
}
-EXPORT_SYMBOL(sn_dma_mapping_error);
+
+u64 sn_dma_get_required_mask(struct device *dev)
+{
+ return DMA_BIT_MASK(64);
+}
+EXPORT_SYMBOL_GPL(sn_dma_get_required_mask);
char *sn_pci_get_legacy_mem(struct pci_bus *bus)
{
{
unsigned long addr;
int ret;
+ struct ia64_sal_retval isrv;
+
+ /*
+ * First, try the SN_SAL_IOIF_PCI_SAFE SAL call which can work
+ * around hw issues at the pci bus level. SGI proms older than
+ * 4.10 don't implement this.
+ */
+
+ SAL_CALL(isrv, SN_SAL_IOIF_PCI_SAFE,
+ pci_domain_nr(bus), bus->number,
+ 0, /* io */
+ 0, /* read */
+ port, size, __pa(val));
+
+ if (isrv.status == 0)
+ return size;
+
+ /*
+ * If the above failed, retry using the SAL_PROBE call which should
+ * be present in all proms (but which cannot work round PCI chipset
+ * bugs). This code is retained for compatibility with old
+ * pre-4.10 proms, and should be removed at some point in the future.
+ */
if (!SN_PCIBUS_BUSSOFT(bus))
return -ENODEV;
int ret = size;
unsigned long paddr;
unsigned long *addr;
+ struct ia64_sal_retval isrv;
+
+ /*
+ * First, try the SN_SAL_IOIF_PCI_SAFE SAL call which can work
+ * around hw issues at the pci bus level. SGI proms older than
+ * 4.10 don't implement this.
+ */
+
+ SAL_CALL(isrv, SN_SAL_IOIF_PCI_SAFE,
+ pci_domain_nr(bus), bus->number,
+ 0, /* io */
+ 1, /* write */
+ port, size, __pa(&val));
+
+ if (isrv.status == 0)
+ return size;
+
+ /*
+ * If the above failed, retry using the SAL_PROBE call which should
+ * be present in all proms (but which cannot work round PCI chipset
+ * bugs). This code is retained for compatibility with old
+ * pre-4.10 proms, and should be removed at some point in the future.
+ */
if (!SN_PCIBUS_BUSSOFT(bus)) {
ret = -ENODEV;
out:
return ret;
}
+
+static struct dma_map_ops sn_dma_ops = {
+ .alloc_coherent = sn_dma_alloc_coherent,
+ .free_coherent = sn_dma_free_coherent,
+ .map_page = sn_dma_map_page,
+ .unmap_page = sn_dma_unmap_page,
+ .map_sg = sn_dma_map_sg,
+ .unmap_sg = sn_dma_unmap_sg,
+ .sync_single_for_cpu = sn_dma_sync_single_for_cpu,
+ .sync_sg_for_cpu = sn_dma_sync_sg_for_cpu,
+ .sync_single_for_device = sn_dma_sync_single_for_device,
+ .sync_sg_for_device = sn_dma_sync_sg_for_device,
+ .mapping_error = sn_dma_mapping_error,
+ .dma_supported = sn_dma_supported,
+};
+
+void sn_dma_init(void)
+{
+ dma_ops = &sn_dma_ops;
+}