sparc64: Enable sun4v dma ops to use IOMMU v2 APIs

Add Hypervisor IOMMU v2 APIs pci_iotsb_map(), pci_iotsb_demap() and
enable sun4v dma ops to use IOMMU v2 API for all PCIe devices with
64bit DMA mask.

Orabug: 23239179

Reviewed-by: chris hyser <chris.hyser@oracle.com>
Signed-off-by: Tushar Dave <tushar.n.dave@oracle.com>
Signed-off-by: Allen Pais <allen.pais@oracle.com>

diff --git a/arch/sparc/include/asm/hypervisor.h b/arch/sparc/include/asm/hypervisor.h
index a3dbbcac5f5fef2269b64cdd2bd163ddc76d20ce..9e23c56eebdedccadfd2d7bae2b745c77265e826 100644 (file)
--- a/arch/sparc/include/asm/hypervisor.h
+++ b/arch/sparc/include/asm/hypervisor.h
@@ -2540,6 +2540,12 @@ unsigned long sun4v_vintr_set_target(unsigned long dev_handle,
  *     iotsb_index     Zero-based IOTTE number within an IOTSB.
  */
 
+/* The index_count argument consists of two fields:
+ * bits 63:48 #iottes and bits 47:0 iotsb_index
+ */
+#define HV_PCI_IOTSB_INDEX_COUNT(__iottes, __iotsb_index) \
+       (((u64)(__iottes) << 48UL) | ((u64)(__iotsb_index)))
+
 /* pci_iotsb_conf()
  * TRAP:       HV_FAST_TRAP
  * FUNCTION:   HV_FAST_PCI_IOTSB_CONF

diff --git a/arch/sparc/kernel/pci_sun4v.c b/arch/sparc/kernel/pci_sun4v.c
index a43e70cdb6459e09c29aae83f8f204af44b5830c..db968b72702eb60ea74ba14ab8cf6b2b5410c91b 100644 (file)
--- a/arch/sparc/kernel/pci_sun4v.c
+++ b/arch/sparc/kernel/pci_sun4v.c
@@ -89,32 +89,54 @@ static inline void iommu_batch_start(struct device *dev, unsigned long prot, uns
 }
 
 /* Interrupts must be disabled.  */
-static long iommu_batch_flush(struct iommu_batch *p)
+static long iommu_batch_flush(struct iommu_batch *p, u64 mask)
 {
        struct pci_pbm_info *pbm = p->dev->archdata.host_controller;
+       u64 *pglist = p->pglist;
+       u64 index_count;
        unsigned long devhandle = pbm->devhandle;
        unsigned long prot = p->prot;
        unsigned long entry = p->entry;
-       u64 *pglist = p->pglist;
        unsigned long npages = p->npages;
+       unsigned long iotsb_num;
+       unsigned long ret;
+       long num;
+
+       /* VPCI maj=1, min=[0,1] only supports read and write */
+       if (vpci_major < 2)
+               prot &= (HV_PCI_MAP_ATTR_READ | HV_PCI_MAP_ATTR_WRITE);
 
        while (npages != 0) {
-               long num;
-
-               /* VPCI maj=1, min=[0,1] only supports read and write */
-               if (vpci_major < 2)
-                       prot &= (HV_PCI_MAP_ATTR_READ | HV_PCI_MAP_ATTR_WRITE);
-
-               num = pci_sun4v_iommu_map(devhandle, HV_PCI_TSBID(0, entry),
-                                         npages, prot, __pa(pglist));
-               if (unlikely(num < 0)) {
-                       if (printk_ratelimit())
-                               printk("iommu_batch_flush: IOMMU map of "
-                                      "[%08lx:%08llx:%lx:%lx:%lx] failed with "
-                                      "status %ld\n",
-                                      devhandle, HV_PCI_TSBID(0, entry),
-                                      npages, prot, __pa(pglist), num);
-                       return -1;
+               if (mask <= DMA_BIT_MASK(32)) {
+                       num = pci_sun4v_iommu_map(devhandle,
+                                                 HV_PCI_TSBID(0, entry),
+                                                 npages,
+                                                 prot,
+                                                 __pa(pglist));
+                       if (unlikely(num < 0)) {
+                               pr_err_ratelimited("iommu_batch_flush: IOMMU map of [%08lx:%08llx:%lx:%lx:%lx] failed with status %ld\n",
+                                                  devhandle,
+                                                  HV_PCI_TSBID(0, entry),
+                                                  npages, prot, __pa(pglist),
+                                                  num);
+                               return -1;
+                       }
+               } else {
+                       index_count = HV_PCI_IOTSB_INDEX_COUNT(npages, entry),
+                       iotsb_num = pbm->iommu->atu->iotsb->iotsb_num;
+                       ret = pci_sun4v_iotsb_map(devhandle,
+                                                 iotsb_num,
+                                                 index_count,
+                                                 prot,
+                                                 __pa(pglist),
+                                                 &num);
+                       if (unlikely(ret != HV_EOK)) {
+                               pr_err_ratelimited("iommu_batch_flush: ATU map of [%08lx:%lx:%llx:%lx:%lx] failed with status %ld\n",
+                                                  devhandle, iotsb_num,
+                                                  index_count, prot,
+                                                  __pa(pglist), ret);
+                               return -1;
+                       }
                }
 
                entry += num;
@@ -128,19 +150,19 @@ static long iommu_batch_flush(struct iommu_batch *p)
        return 0;
 }
 
-static inline void iommu_batch_new_entry(unsigned long entry)
+static inline void iommu_batch_new_entry(unsigned long entry, u64 mask)
 {
        struct iommu_batch *p = this_cpu_ptr(&iommu_batch);
 
        if (p->entry + p->npages == entry)
                return;
        if (p->entry != ~0UL)
-               iommu_batch_flush(p);
+               iommu_batch_flush(p, mask);
        p->entry = entry;
 }
 
 /* Interrupts must be disabled.  */
-static inline long iommu_batch_add(u64 phys_page)
+static inline long iommu_batch_add(u64 phys_page, u64 mask)
 {
        struct iommu_batch *p = this_cpu_ptr(&iommu_batch);
 
@@ -148,28 +170,31 @@ static inline long iommu_batch_add(u64 phys_page)
 
        p->pglist[p->npages++] = phys_page;
        if (p->npages == PGLIST_NENTS)
-               return iommu_batch_flush(p);
+               return iommu_batch_flush(p, mask);
 
        return 0;
 }
 
 /* Interrupts must be disabled.  */
-static inline long iommu_batch_end(void)
+static inline long iommu_batch_end(u64 mask)
 {
        struct iommu_batch *p = this_cpu_ptr(&iommu_batch);
 
        BUG_ON(p->npages >= PGLIST_NENTS);
 
-       return iommu_batch_flush(p);
+       return iommu_batch_flush(p, mask);
 }
 
 static void *dma_4v_alloc_coherent(struct device *dev, size_t size,
                                   dma_addr_t *dma_addrp, gfp_t gfp,
                                   struct dma_attrs *attrs)
 {
+       u64 mask;
        unsigned long flags, order, first_page, npages, n;
        unsigned long prot = 0;
        struct iommu *iommu;
+       struct atu *atu;
+       struct iommu_map_table *tbl;
        struct page *page;
        void *ret;
        long entry;
@@ -194,14 +219,21 @@ static void *dma_4v_alloc_coherent(struct device *dev, size_t size,
        memset((char *)first_page, 0, PAGE_SIZE << order);
 
        iommu = dev->archdata.iommu;
+       atu = iommu->atu;
+
+       mask = dev->coherent_dma_mask;
+       if (mask <= DMA_BIT_MASK(32))
+               tbl = &iommu->tbl;
+       else
+               tbl = &atu->tbl;
 
-       entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages, NULL,
+       entry = iommu_tbl_range_alloc(dev, tbl, npages, NULL,
                                      (unsigned long)(-1), 0);
 
        if (unlikely(entry == IOMMU_ERROR_CODE))
                goto range_alloc_fail;
 
-       *dma_addrp = (iommu->tbl.table_map_base + (entry << IO_PAGE_SHIFT));
+       *dma_addrp = (tbl->table_map_base + (entry << IO_PAGE_SHIFT));
        ret = (void *) first_page;
        first_page = __pa(first_page);
 
@@ -213,12 +245,12 @@ static void *dma_4v_alloc_coherent(struct device *dev, size_t size,
                          entry);
 
        for (n = 0; n < npages; n++) {
-               long err = iommu_batch_add(first_page + (n * PAGE_SIZE));
+               long err = iommu_batch_add(first_page + (n * PAGE_SIZE), mask);
                if (unlikely(err < 0L))
                        goto iommu_map_fail;
        }
 
-       if (unlikely(iommu_batch_end() < 0L))
+       if (unlikely(iommu_batch_end(mask) < 0L))
                goto iommu_map_fail;
 
        local_irq_restore(flags);
@@ -226,7 +258,7 @@ static void *dma_4v_alloc_coherent(struct device *dev, size_t size,
        return ret;
 
 iommu_map_fail:
-       iommu_tbl_range_free(&iommu->tbl, *dma_addrp, npages, IOMMU_ERROR_CODE);
+       iommu_tbl_range_free(tbl, *dma_addrp, npages, IOMMU_ERROR_CODE);
 
 range_alloc_fail:
        free_pages(first_page, order);
@@ -270,18 +302,27 @@ unsigned long dma_4v_iotsb_bind(unsigned long devhandle,
        return 0;
 }
 
-static void dma_4v_iommu_demap(void *demap_arg, unsigned long entry,
-                              unsigned long npages)
+static void dma_4v_iommu_demap(struct device *dev, unsigned long devhandle,
+                              dma_addr_t dvma, unsigned long iotsb_num,
+                              unsigned long entry, unsigned long npages)
 {
-       u32 devhandle = *(u32 *)demap_arg;
        unsigned long num, flags;
+       unsigned long ret;
 
        local_irq_save(flags);
        do {
-               num = pci_sun4v_iommu_demap(devhandle,
-                                           HV_PCI_TSBID(0, entry),
-                                           npages);
-
+               if (dvma <= DMA_BIT_MASK(32)) {
+                       num = pci_sun4v_iommu_demap(devhandle,
+                                                   HV_PCI_TSBID(0, entry),
+                                                   npages);
+               } else {
+                       ret = pci_sun4v_iotsb_demap(devhandle, iotsb_num,
+                                                   entry, npages, &num);
+                       if (unlikely(ret != HV_EOK)) {
+                               pr_err_ratelimited("pci_iotsb_demap() failed with error: %ld\n",
+                                                  ret);
+                       }
+               }
                entry += num;
                npages -= num;
        } while (npages != 0);
@@ -293,16 +334,28 @@ static void dma_4v_free_coherent(struct device *dev, size_t size, void *cpu,
 {
        struct pci_pbm_info *pbm;
        struct iommu *iommu;
+       struct atu *atu;
+       struct iommu_map_table *tbl;
        unsigned long order, npages, entry;
+       unsigned long iotsb_num;
        u32 devhandle;
 
        npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT;
        iommu = dev->archdata.iommu;
        pbm = dev->archdata.host_controller;
+       atu = iommu->atu;
        devhandle = pbm->devhandle;
-       entry = ((dvma - iommu->tbl.table_map_base) >> IO_PAGE_SHIFT);
-       dma_4v_iommu_demap(&devhandle, entry, npages);
-       iommu_tbl_range_free(&iommu->tbl, dvma, npages, IOMMU_ERROR_CODE);
+
+       if (dvma <= DMA_BIT_MASK(32)) {
+               tbl = &iommu->tbl;
+               iotsb_num = 0; /*we don't care for legacy iommu */
+       } else {
+               tbl = &atu->tbl;
+               iotsb_num = atu->iotsb->iotsb_num;
+       }
+       entry = ((dvma - tbl->table_map_base) >> IO_PAGE_SHIFT);
+       dma_4v_iommu_demap(dev, devhandle, dvma, iotsb_num, entry, npages);
+       iommu_tbl_range_free(tbl, dvma, npages, IOMMU_ERROR_CODE);
        order = get_order(size);
        if (order < 10)
                free_pages((unsigned long)cpu, order);
@@ -315,10 +368,13 @@ static dma_addr_t dma_4v_map_page(struct device *dev, struct page *page,
 {
        struct pci_dev *pdev = to_pci_dev(dev);
        struct iommu *iommu;
+       struct atu *atu;
+       struct iommu_map_table *tbl;
+       u64 mask;
        unsigned long flags, npages, oaddr;
        unsigned long i, base_paddr;
-       u32 bus_addr, ret;
        unsigned long prot;
+       dma_addr_t bus_addr, ret;
        long entry;
 
        if (IS_IB_DEVICE(pdev))
@@ -326,6 +382,7 @@ static dma_addr_t dma_4v_map_page(struct device *dev, struct page *page,
                                              direction, attrs);
 
        iommu = dev->archdata.iommu;
+       atu = iommu->atu;
 
        if (unlikely(direction == DMA_NONE))
                goto bad;
@@ -334,13 +391,19 @@ static dma_addr_t dma_4v_map_page(struct device *dev, struct page *page,
        npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK);
        npages >>= IO_PAGE_SHIFT;
 
-       entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages, NULL,
+       mask = *dev->dma_mask;
+       if (mask <= DMA_BIT_MASK(32))
+               tbl = &iommu->tbl;
+       else
+               tbl = &atu->tbl;
+
+       entry = iommu_tbl_range_alloc(dev, tbl, npages, NULL,
                                      (unsigned long)(-1), 0);
 
        if (unlikely(entry == IOMMU_ERROR_CODE))
                goto bad;
 
-       bus_addr = (iommu->tbl.table_map_base + (entry << IO_PAGE_SHIFT));
+       bus_addr = (tbl->table_map_base + (entry << IO_PAGE_SHIFT));
        ret = bus_addr | (oaddr & ~IO_PAGE_MASK);
        base_paddr = __pa(oaddr & IO_PAGE_MASK);
        prot = HV_PCI_MAP_ATTR_READ;
@@ -355,11 +418,11 @@ static dma_addr_t dma_4v_map_page(struct device *dev, struct page *page,
        iommu_batch_start(dev, prot, entry);
 
        for (i = 0; i < npages; i++, base_paddr += IO_PAGE_SIZE) {
-               long err = iommu_batch_add(base_paddr);
+               long err = iommu_batch_add(base_paddr, mask);
                if (unlikely(err < 0L))
                        goto iommu_map_fail;
        }
-       if (unlikely(iommu_batch_end() < 0L))
+       if (unlikely(iommu_batch_end(mask) < 0L))
                goto iommu_map_fail;
 
        local_irq_restore(flags);
@@ -372,7 +435,7 @@ bad:
        return DMA_ERROR_CODE;
 
 iommu_map_fail:
-       iommu_tbl_range_free(&iommu->tbl, bus_addr, npages, IOMMU_ERROR_CODE);
+       iommu_tbl_range_free(tbl, bus_addr, npages, IOMMU_ERROR_CODE);
        return DMA_ERROR_CODE;
 }
 
@@ -383,7 +446,10 @@ static void dma_4v_unmap_page(struct device *dev, dma_addr_t bus_addr,
        struct pci_pbm_info *pbm;
        struct pci_dev *pdev = to_pci_dev(dev);
        struct iommu *iommu;
+       struct atu *atu;
+       struct iommu_map_table *tbl;
        unsigned long npages;
+       unsigned long iotsb_num;
        long entry;
        u32 devhandle;
 
@@ -399,14 +465,23 @@ static void dma_4v_unmap_page(struct device *dev, dma_addr_t bus_addr,
 
        iommu = dev->archdata.iommu;
        pbm = dev->archdata.host_controller;
+       atu = iommu->atu;
        devhandle = pbm->devhandle;
 
        npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK);
        npages >>= IO_PAGE_SHIFT;
        bus_addr &= IO_PAGE_MASK;
-       entry = (bus_addr - iommu->tbl.table_map_base) >> IO_PAGE_SHIFT;
-       dma_4v_iommu_demap(&devhandle, entry, npages);
-       iommu_tbl_range_free(&iommu->tbl, bus_addr, npages, IOMMU_ERROR_CODE);
+
+       if (bus_addr <= DMA_BIT_MASK(32)) {
+               iotsb_num = 0; /* we don't care for legacy iommu */
+               tbl = &iommu->tbl;
+       } else {
+               iotsb_num = atu->iotsb->iotsb_num;
+               tbl = &atu->tbl;
+       }
+       entry = (bus_addr - tbl->table_map_base) >> IO_PAGE_SHIFT;
+       dma_4v_iommu_demap(dev, devhandle, bus_addr, iotsb_num, entry, npages);
+       iommu_tbl_range_free(tbl, bus_addr, npages, IOMMU_ERROR_CODE);
 }
 
 static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist,
@@ -421,6 +496,9 @@ static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist,
        unsigned long seg_boundary_size;
        int outcount, incount, i;
        struct iommu *iommu;
+       struct atu *atu;
+       struct iommu_map_table *tbl;
+       u64 mask;
        unsigned long base_shift;
        long err;
 
@@ -431,6 +509,8 @@ static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist,
        BUG_ON(direction == DMA_NONE);
 
        iommu = dev->archdata.iommu;
+       atu = iommu->atu;
+
        if (nelems == 0 || !iommu)
                return 0;
 
@@ -456,7 +536,15 @@ static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist,
        max_seg_size = dma_get_max_seg_size(dev);
        seg_boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
                                  IO_PAGE_SIZE) >> IO_PAGE_SHIFT;
-       base_shift = iommu->tbl.table_map_base >> IO_PAGE_SHIFT;
+
+       mask = *dev->dma_mask;
+       if (mask <= DMA_BIT_MASK(32))
+               tbl = &iommu->tbl;
+       else
+               tbl = &atu->tbl;
+
+       base_shift = tbl->table_map_base >> IO_PAGE_SHIFT;
+
        for_each_sg(sglist, s, nelems, i) {
                unsigned long paddr, npages, entry, out_entry = 0, slen;
 
@@ -469,27 +557,26 @@ static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist,
                /* Allocate iommu entries for that segment */
                paddr = (unsigned long) SG_ENT_PHYS_ADDRESS(s);
                npages = iommu_num_pages(paddr, slen, IO_PAGE_SIZE);
-               entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages,
+               entry = iommu_tbl_range_alloc(dev, tbl, npages,
                                              &handle, (unsigned long)(-1), 0);
 
                /* Handle failure */
                if (unlikely(entry == IOMMU_ERROR_CODE)) {
-                       if (printk_ratelimit())
-                               printk(KERN_INFO "iommu_alloc failed, iommu %p paddr %lx"
-                                      " npages %lx\n", iommu, paddr, npages);
+                       pr_err_ratelimited("iommu_alloc failed, iommu %p paddr %lx npages %lx\n",
+                                          tbl, paddr, npages);
                        goto iommu_map_failed;
                }
 
-               iommu_batch_new_entry(entry);
+               iommu_batch_new_entry(entry, mask);
 
                /* Convert entry to a dma_addr_t */
-               dma_addr = iommu->tbl.table_map_base + (entry << IO_PAGE_SHIFT);
+               dma_addr = tbl->table_map_base + (entry << IO_PAGE_SHIFT);
                dma_addr |= (s->offset & ~IO_PAGE_MASK);
 
                /* Insert into HW table */
                paddr &= IO_PAGE_MASK;
                while (npages--) {
-                       err = iommu_batch_add(paddr);
+                       err = iommu_batch_add(paddr, mask);
                        if (unlikely(err < 0L))
                                goto iommu_map_failed;
                        paddr += IO_PAGE_SIZE;
@@ -524,7 +611,7 @@ static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist,
                dma_next = dma_addr + slen;
        }
 
-       err = iommu_batch_end();
+       err = iommu_batch_end(mask);
 
        if (unlikely(err < 0L))
                goto iommu_map_failed;
@@ -547,7 +634,7 @@ iommu_map_failed:
                        vaddr = s->dma_address & IO_PAGE_MASK;
                        npages = iommu_num_pages(s->dma_address, s->dma_length,
                                                 IO_PAGE_SIZE);
-                       iommu_tbl_range_free(&iommu->tbl, vaddr, npages,
+                       iommu_tbl_range_free(tbl, vaddr, npages,
                                             IOMMU_ERROR_CODE);
                        /* XXX demap? XXX */
                        s->dma_address = DMA_ERROR_CODE;
@@ -569,7 +656,9 @@ static void dma_4v_unmap_sg(struct device *dev, struct scatterlist *sglist,
        struct pci_dev *pdev = to_pci_dev(dev);
        struct scatterlist *sg;
        struct iommu *iommu;
+       struct atu *atu;
        unsigned long flags, entry;
+       unsigned long iotsb_num;
        u32 devhandle;
 
        /* IB uses bypass, no need to unmap for bypass */
@@ -580,6 +669,7 @@ static void dma_4v_unmap_sg(struct device *dev, struct scatterlist *sglist,
 
        iommu = dev->archdata.iommu;
        pbm = dev->archdata.host_controller;
+       atu = iommu->atu;
        devhandle = pbm->devhandle;
        
        local_irq_save(flags);
@@ -589,15 +679,24 @@ static void dma_4v_unmap_sg(struct device *dev, struct scatterlist *sglist,
                dma_addr_t dma_handle = sg->dma_address;
                unsigned int len = sg->dma_length;
                unsigned long npages;
-               struct iommu_map_table *tbl = &iommu->tbl;
+               struct iommu_map_table *tbl;
                unsigned long shift = IO_PAGE_SHIFT;
 
                if (!len)
                        break;
                npages = iommu_num_pages(dma_handle, len, IO_PAGE_SIZE);
+
+               if (dma_handle <= DMA_BIT_MASK(32)) {
+                       iotsb_num = 0; /* we don't care for legacy iommu */
+                       tbl = &iommu->tbl;
+               } else {
+                       iotsb_num = atu->iotsb->iotsb_num;
+                       tbl = &atu->tbl;
+               }
                entry = ((dma_handle - tbl->table_map_base) >> shift);
-               dma_4v_iommu_demap(&devhandle, entry, npages);
-               iommu_tbl_range_free(&iommu->tbl, dma_handle, npages,
+               dma_4v_iommu_demap(dev, devhandle, dma_handle, iotsb_num,
+                                  entry, npages);
+               iommu_tbl_range_free(tbl, dma_handle, npages,
                                     IOMMU_ERROR_CODE);
                sg = sg_next(sg);
        }

diff --git a/arch/sparc/kernel/pci_sun4v.h b/arch/sparc/kernel/pci_sun4v.h
index 25826c6e3abea66b9e5c1f3ac121d4e1128bbdd2..c87db0a89149d816cb70a12090b38a980e8f9d90 100644 (file)
--- a/arch/sparc/kernel/pci_sun4v.h
+++ b/arch/sparc/kernel/pci_sun4v.h
@@ -102,5 +102,16 @@ unsigned long pci_sun4v_iotsb_conf(unsigned long devhandle,
 unsigned long pci_sun4v_iotsb_bind(unsigned long devhandle,
                                   unsigned long iotsb_num,
                                   unsigned int pci_device);
+unsigned long pci_sun4v_iotsb_map(unsigned long devhandle,
+                                 unsigned long iotsb_num,
+                                 unsigned long iotsb_index_iottes,
+                                 unsigned long io_attributes,
+                                 unsigned long io_page_list_pa,
+                                 long *mapped);
+unsigned long pci_sun4v_iotsb_demap(unsigned long devhandle,
+                                   unsigned long iotsb_num,
+                                   unsigned long iotsb_index,
+                                   unsigned long iottes,
+                                   unsigned long *demapped);
 
 #endif /* !(_PCI_SUN4V_H) */

diff --git a/arch/sparc/kernel/pci_sun4v_asm.S b/arch/sparc/kernel/pci_sun4v_asm.S
index 00e55efdc379ed212430bb7bd87f7f10457d89bb..55eb534e4613b4ba8a6be5486a35c7d6ea51ca34 100644 (file)
--- a/arch/sparc/kernel/pci_sun4v_asm.S
+++ b/arch/sparc/kernel/pci_sun4v_asm.S
@@ -407,3 +407,39 @@ ENTRY(pci_sun4v_iotsb_bind)
        retl
        nop
 ENDPROC(pci_sun4v_iotsb_bind)
+
+       /*
+        * %o0: devhandle
+        * %o1: iotsb
+        * %o2: iotsb_index_and_#iottes
+        *        63:48 - #iottes (unsigned integer 16 bits)
+        *        47:00 - iotsb_index (unsigned integer 48 bits)
+        * %o3: iotte_attributes
+        * %o4: io_page_list_p
+        *
+        * returns %o0: status
+        *         %o1: #iottes_mapped
+        */
+ENTRY(pci_sun4v_iotsb_map)
+       mov     %o5, %g1
+       mov     HV_FAST_PCI_IOTSB_MAP, %o5
+       ta      HV_FAST_TRAP
+       retl
+       stx     %o1, [%g1]      /* #mapped */
+ENDPROC(pci_sun4v_iotsb_map)
+
+       /*
+        * %o0: devhandle
+        * %o1: iotsb
+        * %o2: iotsb_index
+        * %o3: #iottes
+        *
+        * returns %o0: status
+        *         %o1: #iottes_demapped
+        */
+ENTRY(pci_sun4v_iotsb_demap)
+       mov     HV_FAST_PCI_IOTSB_DEMAP, %o5
+       ta      HV_FAST_TRAP
+       retl
+       stx     %o1, [%o4]      /* #demapped */
+ENDPROC(pci_sun4v_iotsb_demap)