dmaengine: dw-edma: Add CPU to PCI bus address translation

Since 9575632052ba ("dmaengine: make slave address physical"), the source
and destination addresses of the DMA slave device have been converted to
physical addresses in the CPU address space. It's the DMA device driver's
responsibility to convert them to the DMA bus address space. In case of the
DW eDMA device, the source or destination peripheral (slave) devices reside
in PCI bus space. Thus we need to perform the PCI Host/Endpoint windows-
based (i.e. DT "ranges" property) address translation; otherwise the eDMA
transactions won't work as expected (or can be even harmful) if the CPU and
PCI address spaces don't match.

Note 1: Even though the DMA interleaved template has both source and
destination addresses declared as dma_addr_t, only the CPU memory range
should be mapped to be seen by the DMA device since it's a subject of the
DMA getting towards the system side. The device part must not be mapped
since the slave device resides in the PCI bus space, which isn't affected
by IOMMUs or iATU translations. DW PCIe eDMA generates corresponding
MWr/MRd TLPs on its own.

Note 2: This functionality is mainly required for the remote eDMA setup
since the CPU address must be manually translated into the PCI bus space
before being written to LLI.{SAR,DAR}. If eDMA is embedded in the locally
accessible DW PCIe Root Port/Endpoint, software-based translation isn't
required since hardware will translate it via the Outbound iATU as long as
the DMA_BYPASS flag is cleared. If DMA_BYPASS is set or there is no
Outbound iATU entry that contains the SAR or DAR (for Read and Write
channel respectively), there won't be any translation performed but DMA
will proceed with the corresponding source/destination address as-is.

Link: https://lore.kernel.org/r/20230113171409.30470-8-Sergey.Semin@baikalelectronics.ru
Tested-by: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
Signed-off-by: Serge Semin <Sergey.Semin@baikalelectronics.ru>
Signed-off-by: Lorenzo Pieralisi <lpieralisi@kernel.org>
Signed-off-by: Bjorn Helgaas <bhelgaas@google.com>
Reviewed-by: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>
Acked-by: Vinod Koul <vkoul@kernel.org>

diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c
index d5c4192141ef319025ad0a50f01f973b134c74c2..6c9f95a8e39721c7fc87ab8c9d10c2788301ead0 100644 (file)
--- a/drivers/dma/dw-edma/dw-edma-core.c
+++ b/drivers/dma/dw-edma/dw-edma-core.c
@@ -39,6 +39,17 @@ struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd)
        return container_of(vd, struct dw_edma_desc, vd);
 }
 
+static inline
+u64 dw_edma_get_pci_address(struct dw_edma_chan *chan, phys_addr_t cpu_addr)
+{
+       struct dw_edma_chip *chip = chan->dw->chip;
+
+       if (chip->ops->pci_address)
+               return chip->ops->pci_address(chip->dev, cpu_addr);
+
+       return cpu_addr;
+}
+
 static struct dw_edma_burst *dw_edma_alloc_burst(struct dw_edma_chunk *chunk)
 {
        struct dw_edma_burst *burst;
@@ -327,11 +338,11 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer)
 {
        struct dw_edma_chan *chan = dchan2dw_edma_chan(xfer->dchan);
        enum dma_transfer_direction dir = xfer->direction;
-       phys_addr_t src_addr, dst_addr;
        struct scatterlist *sg = NULL;
        struct dw_edma_chunk *chunk;
        struct dw_edma_burst *burst;
        struct dw_edma_desc *desc;
+       u64 src_addr, dst_addr;
        size_t fsz = 0;
        u32 cnt = 0;
        int i;
@@ -406,6 +417,11 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer)
                dst_addr = chan->config.dst_addr;
        }
 
+       if (dir == DMA_DEV_TO_MEM)
+               src_addr = dw_edma_get_pci_address(chan, (phys_addr_t)src_addr);
+       else
+               dst_addr = dw_edma_get_pci_address(chan, (phys_addr_t)dst_addr);
+
        if (xfer->type == EDMA_XFER_CYCLIC) {
                cnt = xfer->xfer.cyclic.cnt;
        } else if (xfer->type == EDMA_XFER_SCATTER_GATHER) {

diff --git a/include/linux/dma/edma.h b/include/linux/dma/edma.h
index a864978ddd27dc40e230ed7b04e2d03ece113edc..07a23ecc834f58805a7a93919949365abf81b4c1 100644 (file)
--- a/include/linux/dma/edma.h
+++ b/include/linux/dma/edma.h
@@ -23,8 +23,23 @@ struct dw_edma_region {
        size_t          sz;
 };
 
+/**
+ * struct dw_edma_core_ops - platform-specific eDMA methods
+ * @irq_vector:                Get IRQ number of the passed eDMA channel. Note the
+ *                     method accepts the channel id in the end-to-end
+ *                     numbering with the eDMA write channels being placed
+ *                     first in the row.
+ * @pci_address:       Get PCIe bus address corresponding to the passed CPU
+ *                     address. Note there is no need in specifying this
+ *                     function if the address translation is performed by
+ *                     the DW PCIe RP/EP controller with the DW eDMA device in
+ *                     subject and DMA_BYPASS isn't set for all the outbound
+ *                     iATU windows. That will be done by the controller
+ *                     automatically.
+ */
 struct dw_edma_core_ops {
        int (*irq_vector)(struct device *dev, unsigned int nr);
+       u64 (*pci_address)(struct device *dev, phys_addr_t cpu_addr);
 };
 
 enum dw_edma_map_format {