lkml.org 
[lkml]   [2021]   [Dec]   [20]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    /
    SubjectRe: [PATCH 4/5] iommu: Separate IOVA rcache memories from iova_domain structure
    From
    Date
    On 24/09/2021 11:01, John Garry wrote:
    > Only dma-iommu.c and vdpa actually use the "fast" mode of IOVA alloc and
    > free. As such, it's wasteful that all other IOVA domains hold the rcache
    > memories.
    >
    > In addition, the current IOVA domain init implementation is poor
    > (init_iova_domain()), in that errors are ignored and not passed to the
    > caller. The only errors can come from the IOVA rcache init, and fixing up
    > all the IOVA domain init callsites to handle the errors would take some
    > work.
    >
    > Separate the IOVA rache out of the IOVA domain, and create a new IOVA
    > domain structure, iova_caching_domain.
    >
    > Signed-off-by: John Garry <john.garry@huawei.com>

    Hi Robin,

    Do you have any thoughts on this patch? The decision is whether we stick
    with a single iova domain structure or support this super structure for
    iova domains which support the rcache. I did not try the former - it
    would be do-able but I am not sure on how it would look.

    Thanks,
    John

    EOM

    > ---
    > drivers/iommu/dma-iommu.c | 56 +++++++-----
    > drivers/iommu/iova.c | 125 ++++++++++++++++++---------
    > drivers/vdpa/vdpa_user/iova_domain.c | 53 +++++++-----
    > drivers/vdpa/vdpa_user/iova_domain.h | 4 +-
    > include/linux/iova.h | 18 ++--
    > 5 files changed, 166 insertions(+), 90 deletions(-)
    >
    > diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
    > index fd669bad96e1..70651f1a688d 100644
    > --- a/drivers/iommu/dma-iommu.c
    > +++ b/drivers/iommu/dma-iommu.c
    > @@ -92,8 +92,8 @@ struct iommu_dma_cookie {
    > union {
    > /* Full allocator for IOMMU_DMA_IOVA_COOKIE */
    > struct {
    > - struct iova_domain iovad;
    > - struct fq_domain fq;
    > + struct iova_caching_domain rcached;
    > + struct fq_domain fq;
    > };
    > /* Trivial linear page allocator for IOMMU_DMA_MSI_COOKIE */
    > dma_addr_t msi_iova;
    > @@ -197,7 +197,6 @@ static void fq_ring_free(struct fq_domain *fq_domain, struct fq *fq)
    > struct iommu_dma_cookie *cookie = container_of(fq_domain,
    > struct iommu_dma_cookie,
    > fq);
    > - struct iova_domain *iovad = &cookie->iovad;
    > u64 counter = atomic64_read(&fq_domain->fq_flush_finish_cnt);
    > unsigned idx;
    >
    > @@ -211,7 +210,7 @@ static void fq_ring_free(struct fq_domain *fq_domain, struct fq *fq)
    > if (fq_domain->entry_dtor)
    > fq_domain->entry_dtor(fq->entries[idx].data);
    >
    > - free_iova_fast(iovad,
    > + free_iova_fast(&cookie->rcached,
    > fq->entries[idx].iova_pfn,
    > fq->entries[idx].pages);
    >
    > @@ -330,7 +329,7 @@ static int init_flush_queue(struct fq_domain *fq_domain,
    > static inline size_t cookie_msi_granule(struct iommu_dma_cookie *cookie)
    > {
    > if (cookie->type == IOMMU_DMA_IOVA_COOKIE)
    > - return cookie->iovad.granule;
    > + return cookie->rcached.iovad.granule;
    > return PAGE_SIZE;
    > }
    >
    > @@ -413,9 +412,10 @@ void iommu_put_dma_cookie(struct iommu_domain *domain)
    > if (!cookie)
    > return;
    >
    > - if (cookie->type == IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule) {
    > + if (cookie->type == IOMMU_DMA_IOVA_COOKIE &&
    > + cookie->rcached.iovad.granule) {
    > free_flush_queue(&cookie->fq);
    > - put_iova_domain(&cookie->iovad);
    > + put_iova_caching_domain(&cookie->rcached);
    > }
    >
    > list_for_each_entry_safe(msi, tmp, &cookie->msi_page_list, list) {
    > @@ -449,7 +449,7 @@ EXPORT_SYMBOL(iommu_dma_get_resv_regions);
    > static int cookie_init_hw_msi_region(struct iommu_dma_cookie *cookie,
    > phys_addr_t start, phys_addr_t end)
    > {
    > - struct iova_domain *iovad = &cookie->iovad;
    > + struct iova_domain *iovad = &cookie->rcached.iovad;
    > struct iommu_dma_msi_page *msi_page;
    > int i, num_pages;
    >
    > @@ -520,7 +520,8 @@ static int iova_reserve_iommu_regions(struct device *dev,
    > struct iommu_domain *domain)
    > {
    > struct iommu_dma_cookie *cookie = domain->iova_cookie;
    > - struct iova_domain *iovad = &cookie->iovad;
    > + struct iova_caching_domain *rcached = &cookie->rcached;
    > + struct iova_domain *iovad = &rcached->iovad;
    > struct iommu_resv_region *region;
    > LIST_HEAD(resv_regions);
    > int ret = 0;
    > @@ -612,14 +613,17 @@ static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base,
    > dma_addr_t limit, struct device *dev)
    > {
    > struct iommu_dma_cookie *cookie = domain->iova_cookie;
    > + struct iova_caching_domain *rcached;
    > unsigned long order, base_pfn;
    > struct iova_domain *iovad;
    > struct fq_domain *fq;
    > + int ret;
    >
    > if (!cookie || cookie->type != IOMMU_DMA_IOVA_COOKIE)
    > return -EINVAL;
    >
    > - iovad = &cookie->iovad;
    > + rcached = &cookie->rcached;
    > + iovad = &rcached->iovad;
    > fq = &cookie->fq;
    >
    > /* Use the smallest supported page size for IOVA granularity */
    > @@ -652,7 +656,11 @@ static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base,
    > fq->flush_cb = NULL;
    > fq->fq = NULL;
    >
    > - init_iova_domain(iovad, 1UL << order, base_pfn);
    > + ret = init_iova_caching_domain(rcached, 1UL << order, base_pfn);
    > + if (ret) {
    > + dev_err(dev, "init_iova_caching_domain failed (%d)\n", ret);
    > + return ret;
    > + }
    >
    > /* If the FQ fails we can simply fall back to strict mode */
    > if (domain->type == IOMMU_DOMAIN_DMA_FQ && iommu_dma_init_fq(domain))
    > @@ -694,13 +702,16 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain,
    > size_t size, u64 dma_limit, struct device *dev)
    > {
    > struct iommu_dma_cookie *cookie = domain->iova_cookie;
    > - struct iova_domain *iovad = &cookie->iovad;
    > + struct iova_caching_domain *rcached;
    > + struct iova_domain *iovad;
    > unsigned long shift, iova_len, iova = 0;
    >
    > if (cookie->type == IOMMU_DMA_MSI_COOKIE) {
    > cookie->msi_iova += size;
    > return cookie->msi_iova - size;
    > }
    > + rcached = &cookie->rcached;
    > + iovad = &rcached->iovad;
    >
    > shift = iova_shift(iovad);
    > iova_len = size >> shift;
    > @@ -712,11 +723,11 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain,
    >
    > /* Try to get PCI devices a SAC address */
    > if (dma_limit > DMA_BIT_MASK(32) && !iommu_dma_forcedac && dev_is_pci(dev))
    > - iova = alloc_iova_fast(iovad, iova_len,
    > + iova = alloc_iova_fast(rcached, iova_len,
    > DMA_BIT_MASK(32) >> shift, false);
    >
    > if (!iova)
    > - iova = alloc_iova_fast(iovad, iova_len, dma_limit >> shift,
    > + iova = alloc_iova_fast(rcached, iova_len, dma_limit >> shift,
    > true);
    >
    > return (dma_addr_t)iova << shift;
    > @@ -725,7 +736,8 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain,
    > static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie,
    > dma_addr_t iova, size_t size, struct iommu_iotlb_gather *gather)
    > {
    > - struct iova_domain *iovad = &cookie->iovad;
    > + struct iova_caching_domain *rcached = &cookie->rcached;
    > + struct iova_domain *iovad = &rcached->iovad;
    >
    > /* The MSI case is only ever cleaning up its most recent allocation */
    > if (cookie->type == IOMMU_DMA_MSI_COOKIE) {
    > @@ -735,7 +747,7 @@ static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie,
    > size >> iova_shift(iovad),
    > (unsigned long)gather->freelist);
    > } else {
    > - free_iova_fast(iovad, iova_pfn(iovad, iova),
    > + free_iova_fast(rcached, iova_pfn(iovad, iova),
    > size >> iova_shift(iovad));
    > }
    > }
    > @@ -745,7 +757,7 @@ static void __iommu_dma_unmap(struct device *dev, dma_addr_t dma_addr,
    > {
    > struct iommu_domain *domain = iommu_get_dma_domain(dev);
    > struct iommu_dma_cookie *cookie = domain->iova_cookie;
    > - struct iova_domain *iovad = &cookie->iovad;
    > + struct iova_domain *iovad = &cookie->rcached.iovad;
    > size_t iova_off = iova_offset(iovad, dma_addr);
    > struct iommu_iotlb_gather iotlb_gather;
    > size_t unmapped;
    > @@ -785,7 +797,7 @@ static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,
    > {
    > struct iommu_domain *domain = iommu_get_dma_domain(dev);
    > struct iommu_dma_cookie *cookie = domain->iova_cookie;
    > - struct iova_domain *iovad = &cookie->iovad;
    > + struct iova_domain *iovad = &cookie->rcached.iovad;
    > size_t iova_off = iova_offset(iovad, phys);
    > dma_addr_t iova;
    >
    > @@ -813,7 +825,7 @@ static dma_addr_t __iommu_dma_map_swiotlb(struct device *dev, phys_addr_t phys,
    > int prot = dma_info_to_prot(dir, coherent, attrs);
    > struct iommu_domain *domain = iommu_get_dma_domain(dev);
    > struct iommu_dma_cookie *cookie = domain->iova_cookie;
    > - struct iova_domain *iovad = &cookie->iovad;
    > + struct iova_domain *iovad = &cookie->rcached.iovad;
    > size_t aligned_size = org_size;
    > void *padding_start;
    > size_t padding_size;
    > @@ -924,7 +936,8 @@ static struct page **__iommu_dma_alloc_noncontiguous(struct device *dev,
    > {
    > struct iommu_domain *domain = iommu_get_dma_domain(dev);
    > struct iommu_dma_cookie *cookie = domain->iova_cookie;
    > - struct iova_domain *iovad = &cookie->iovad;
    > + struct iova_caching_domain *rcached = &cookie->rcached;
    > + struct iova_domain *iovad = &rcached->iovad;
    > bool coherent = dev_is_dma_coherent(dev);
    > int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs);
    > unsigned int count, min_size, alloc_sizes = domain->pgsize_bitmap;
    > @@ -1258,7 +1271,8 @@ static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
    > {
    > struct iommu_domain *domain = iommu_get_dma_domain(dev);
    > struct iommu_dma_cookie *cookie = domain->iova_cookie;
    > - struct iova_domain *iovad = &cookie->iovad;
    > + struct iova_caching_domain *rcached = &cookie->rcached;
    > + struct iova_domain *iovad = &rcached->iovad;
    > struct scatterlist *s, *prev = NULL;
    > int prot = dma_info_to_prot(dir, dev_is_dma_coherent(dev), attrs);
    > dma_addr_t iova;
    > diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c
    > index 104fdc9d6c6a..30eb128b1581 100644
    > --- a/drivers/iommu/iova.c
    > +++ b/drivers/iommu/iova.c
    > @@ -15,27 +15,29 @@
    > /* The anchor node sits above the top of the usable address space */
    > #define IOVA_ANCHOR ~0UL
    >
    > -static bool iova_rcache_insert(struct iova_domain *iovad,
    > +static bool iova_rcache_insert(struct iova_caching_domain *rcached,
    > unsigned long pfn,
    > unsigned long size);
    > -static unsigned long iova_rcache_get(struct iova_domain *iovad,
    > +static unsigned long iova_rcache_get(struct iova_caching_domain *rcached,
    > unsigned long size,
    > unsigned long limit_pfn);
    > -static void init_iova_rcaches(struct iova_domain *iovad);
    > -static void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad);
    > -static void free_iova_rcaches(struct iova_domain *iovad);
    > +static int init_iova_rcaches(struct iova_caching_domain *rcached);
    > +static void free_cpu_cached_iovas(unsigned int cpu,
    > + struct iova_caching_domain *rcached);
    > +static void free_iova_rcaches(struct iova_caching_domain *rcached);
    >
    > static int iova_cpuhp_dead(unsigned int cpu, struct hlist_node *node)
    > {
    > - struct iova_domain *iovad;
    > + struct iova_caching_domain *rcached;
    >
    > - iovad = hlist_entry_safe(node, struct iova_domain, cpuhp_dead);
    > + rcached = hlist_entry_safe(node, struct iova_caching_domain,
    > + cpuhp_dead);
    >
    > - free_cpu_cached_iovas(cpu, iovad);
    > + free_cpu_cached_iovas(cpu, rcached);
    > return 0;
    > }
    >
    > -static void free_global_cached_iovas(struct iova_domain *iovad);
    > +static void free_global_cached_iovas(struct iova_caching_domain *rcached);
    >
    > static struct iova *to_iova(struct rb_node *node)
    > {
    > @@ -64,11 +66,32 @@ init_iova_domain(struct iova_domain *iovad, unsigned long granule,
    > iovad->anchor.pfn_lo = iovad->anchor.pfn_hi = IOVA_ANCHOR;
    > rb_link_node(&iovad->anchor.node, NULL, &iovad->rbroot.rb_node);
    > rb_insert_color(&iovad->anchor.node, &iovad->rbroot);
    > - cpuhp_state_add_instance_nocalls(CPUHP_IOMMU_IOVA_DEAD, &iovad->cpuhp_dead);
    > - init_iova_rcaches(iovad);
    > }
    > EXPORT_SYMBOL_GPL(init_iova_domain);
    >
    > +int init_iova_caching_domain(struct iova_caching_domain *rcached,
    > + unsigned long granule, unsigned long start_pfn)
    > +{
    > + int ret;
    > +
    > + ret = cpuhp_state_add_instance_nocalls(CPUHP_IOMMU_IOVA_DEAD,
    > + &rcached->cpuhp_dead);
    > + if (ret)
    > + return ret;
    > +
    > + ret = init_iova_rcaches(rcached);
    > + if (ret) {
    > + cpuhp_state_remove_instance_nocalls(CPUHP_IOMMU_IOVA_DEAD,
    > + &rcached->cpuhp_dead);
    > + return ret;
    > + }
    > +
    > + init_iova_domain(&rcached->iovad, granule, start_pfn);
    > +
    > + return 0;
    > +}
    > +EXPORT_SYMBOL_GPL(init_iova_caching_domain);
    > +
    > static struct rb_node *
    > __get_cached_rbnode(struct iova_domain *iovad, unsigned long limit_pfn)
    > {
    > @@ -422,7 +445,7 @@ EXPORT_SYMBOL_GPL(free_iova);
    >
    > /**
    > * alloc_iova_fast - allocates an iova from rcache
    > - * @iovad: - iova domain in question
    > + * @rcached: - iova caching domain in question
    > * @size: - size of page frames to allocate
    > * @limit_pfn: - max limit address
    > * @flush_rcache: - set to flush rcache on regular allocation failure
    > @@ -431,7 +454,7 @@ EXPORT_SYMBOL_GPL(free_iova);
    > * fails too and the flush_rcache flag is set then the rcache will be flushed.
    > */
    > unsigned long
    > -alloc_iova_fast(struct iova_domain *iovad, unsigned long size,
    > +alloc_iova_fast(struct iova_caching_domain *rcached, unsigned long size,
    > unsigned long limit_pfn, bool flush_rcache)
    > {
    > unsigned long iova_pfn;
    > @@ -446,12 +469,12 @@ alloc_iova_fast(struct iova_domain *iovad, unsigned long size,
    > if (size < (1 << (IOVA_RANGE_CACHE_MAX_SIZE - 1)))
    > size = roundup_pow_of_two(size);
    >
    > - iova_pfn = iova_rcache_get(iovad, size, limit_pfn + 1);
    > + iova_pfn = iova_rcache_get(rcached, size, limit_pfn + 1);
    > if (iova_pfn)
    > return iova_pfn;
    >
    > retry:
    > - new_iova = alloc_iova(iovad, size, limit_pfn, true);
    > + new_iova = alloc_iova(&rcached->iovad, size, limit_pfn, true);
    > if (!new_iova) {
    > unsigned int cpu;
    >
    > @@ -461,8 +484,8 @@ alloc_iova_fast(struct iova_domain *iovad, unsigned long size,
    > /* Try replenishing IOVAs by flushing rcache. */
    > flush_rcache = false;
    > for_each_online_cpu(cpu)
    > - free_cpu_cached_iovas(cpu, iovad);
    > - free_global_cached_iovas(iovad);
    > + free_cpu_cached_iovas(cpu, rcached);
    > + free_global_cached_iovas(rcached);
    > goto retry;
    > }
    >
    > @@ -472,21 +495,22 @@ EXPORT_SYMBOL_GPL(alloc_iova_fast);
    >
    > /**
    > * free_iova_fast - free iova pfn range into rcache
    > - * @iovad: - iova domain in question.
    > + * @rcached: - iova caching domain in question.
    > * @pfn: - pfn that is allocated previously
    > * @size: - # of pages in range
    > * This functions frees an iova range by trying to put it into the rcache,
    > * falling back to regular iova deallocation via free_iova() if this fails.
    > */
    > -void
    > -free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size)
    > +void free_iova_fast(struct iova_caching_domain *rcached, unsigned long pfn,
    > + unsigned long size)
    > {
    > - if (iova_rcache_insert(iovad, pfn, size))
    > + if (iova_rcache_insert(rcached, pfn, size))
    > return;
    >
    > - free_iova(iovad, pfn);
    > + free_iova(&rcached->iovad, pfn);
    > }
    > EXPORT_SYMBOL_GPL(free_iova_fast);
    > +
    > /**
    > * put_iova_domain - destroys the iova domain
    > * @iovad: - iova domain in question.
    > @@ -496,15 +520,23 @@ void put_iova_domain(struct iova_domain *iovad)
    > {
    > struct iova *iova, *tmp;
    >
    > - cpuhp_state_remove_instance_nocalls(CPUHP_IOMMU_IOVA_DEAD,
    > - &iovad->cpuhp_dead);
    > -
    > - free_iova_rcaches(iovad);
    > rbtree_postorder_for_each_entry_safe(iova, tmp, &iovad->rbroot, node)
    > free_iova_mem(iova);
    > }
    > EXPORT_SYMBOL_GPL(put_iova_domain);
    >
    > +void put_iova_caching_domain(struct iova_caching_domain *rcached)
    > +{
    > + struct iova_domain *iovad = &rcached->iovad;
    > +
    > + cpuhp_state_remove_instance_nocalls(CPUHP_IOMMU_IOVA_DEAD,
    > + &rcached->cpuhp_dead);
    > + free_iova_rcaches(rcached);
    > +
    > + put_iova_domain(iovad);
    > +}
    > +EXPORT_SYMBOL_GPL(put_iova_caching_domain);
    > +
    > static int
    > __is_range_overlap(struct rb_node *node,
    > unsigned long pfn_lo, unsigned long pfn_hi)
    > @@ -693,7 +725,7 @@ static void iova_magazine_push(struct iova_magazine *mag, unsigned long pfn)
    > mag->pfns[mag->size++] = pfn;
    > }
    >
    > -static void init_iova_rcaches(struct iova_domain *iovad)
    > +static int init_iova_rcaches(struct iova_caching_domain *rcached)
    > {
    > struct iova_cpu_rcache *cpu_rcache;
    > struct iova_rcache *rcache;
    > @@ -701,12 +733,12 @@ static void init_iova_rcaches(struct iova_domain *iovad)
    > int i;
    >
    > for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
    > - rcache = &iovad->rcaches[i];
    > + rcache = &rcached->rcaches[i];
    > spin_lock_init(&rcache->lock);
    > rcache->depot_size = 0;
    > rcache->cpu_rcaches = __alloc_percpu(sizeof(*cpu_rcache), cache_line_size());
    > - if (WARN_ON(!rcache->cpu_rcaches))
    > - continue;
    > + if (!rcache->cpu_rcaches)
    > + goto err;
    > for_each_possible_cpu(cpu) {
    > cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);
    > spin_lock_init(&cpu_rcache->lock);
    > @@ -714,6 +746,12 @@ static void init_iova_rcaches(struct iova_domain *iovad)
    > cpu_rcache->prev = iova_magazine_alloc(GFP_KERNEL);
    > }
    > }
    > +
    > + return 0;
    > +
    > +err:
    > + free_iova_rcaches(rcached);
    > + return -ENOMEM;
    > }
    >
    > /*
    > @@ -722,7 +760,7 @@ static void init_iova_rcaches(struct iova_domain *iovad)
    > * space, and free_iova() (our only caller) will then return the IOVA
    > * range to the rbtree instead.
    > */
    > -static bool __iova_rcache_insert(struct iova_domain *iovad,
    > +static bool __iova_rcache_insert(struct iova_caching_domain *rcached,
    > struct iova_rcache *rcache,
    > unsigned long iova_pfn)
    > {
    > @@ -763,14 +801,14 @@ static bool __iova_rcache_insert(struct iova_domain *iovad,
    > spin_unlock_irqrestore(&cpu_rcache->lock, flags);
    >
    > if (mag_to_free) {
    > - iova_magazine_free_pfns(mag_to_free, iovad);
    > + iova_magazine_free_pfns(mag_to_free, &rcached->iovad);
    > iova_magazine_free(mag_to_free);
    > }
    >
    > return can_insert;
    > }
    >
    > -static bool iova_rcache_insert(struct iova_domain *iovad, unsigned long pfn,
    > +static bool iova_rcache_insert(struct iova_caching_domain *rcached, unsigned long pfn,
    > unsigned long size)
    > {
    > unsigned int log_size = order_base_2(size);
    > @@ -778,7 +816,7 @@ static bool iova_rcache_insert(struct iova_domain *iovad, unsigned long pfn,
    > if (log_size >= IOVA_RANGE_CACHE_MAX_SIZE)
    > return false;
    >
    > - return __iova_rcache_insert(iovad, &iovad->rcaches[log_size], pfn);
    > + return __iova_rcache_insert(rcached, &rcached->rcaches[log_size], pfn);
    > }
    >
    > /*
    > @@ -825,7 +863,7 @@ static unsigned long __iova_rcache_get(struct iova_rcache *rcache,
    > * size is too big or the DMA limit we are given isn't satisfied by the
    > * top element in the magazine.
    > */
    > -static unsigned long iova_rcache_get(struct iova_domain *iovad,
    > +static unsigned long iova_rcache_get(struct iova_caching_domain *rcached,
    > unsigned long size,
    > unsigned long limit_pfn)
    > {
    > @@ -834,13 +872,13 @@ static unsigned long iova_rcache_get(struct iova_domain *iovad,
    > if (log_size >= IOVA_RANGE_CACHE_MAX_SIZE)
    > return 0;
    >
    > - return __iova_rcache_get(&iovad->rcaches[log_size], limit_pfn - size);
    > + return __iova_rcache_get(&rcached->rcaches[log_size], limit_pfn - size);
    > }
    >
    > /*
    > * free rcache data structures.
    > */
    > -static void free_iova_rcaches(struct iova_domain *iovad)
    > +static void free_iova_rcaches(struct iova_caching_domain *rcached)
    > {
    > struct iova_rcache *rcache;
    > struct iova_cpu_rcache *cpu_rcache;
    > @@ -848,7 +886,7 @@ static void free_iova_rcaches(struct iova_domain *iovad)
    > int i, j;
    >
    > for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
    > - rcache = &iovad->rcaches[i];
    > + rcache = &rcached->rcaches[i];
    > for_each_possible_cpu(cpu) {
    > cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);
    > iova_magazine_free(cpu_rcache->loaded);
    > @@ -863,15 +901,17 @@ static void free_iova_rcaches(struct iova_domain *iovad)
    > /*
    > * free all the IOVA ranges cached by a cpu (used when cpu is unplugged)
    > */
    > -static void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad)
    > +static void free_cpu_cached_iovas(unsigned int cpu,
    > + struct iova_caching_domain *rcached)
    > {
    > + struct iova_domain *iovad = &rcached->iovad;
    > struct iova_cpu_rcache *cpu_rcache;
    > struct iova_rcache *rcache;
    > unsigned long flags;
    > int i;
    >
    > for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
    > - rcache = &iovad->rcaches[i];
    > + rcache = &rcached->rcaches[i];
    > cpu_rcache = per_cpu_ptr(rcache->cpu_rcaches, cpu);
    > spin_lock_irqsave(&cpu_rcache->lock, flags);
    > iova_magazine_free_pfns(cpu_rcache->loaded, iovad);
    > @@ -883,14 +923,15 @@ static void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad)
    > /*
    > * free all the IOVA ranges of global cache
    > */
    > -static void free_global_cached_iovas(struct iova_domain *iovad)
    > +static void free_global_cached_iovas(struct iova_caching_domain *rcached)
    > {
    > + struct iova_domain *iovad = &rcached->iovad;
    > struct iova_rcache *rcache;
    > unsigned long flags;
    > int i, j;
    >
    > for (i = 0; i < IOVA_RANGE_CACHE_MAX_SIZE; ++i) {
    > - rcache = &iovad->rcaches[i];
    > + rcache = &rcached->rcaches[i];
    > spin_lock_irqsave(&rcache->lock, flags);
    > for (j = 0; j < rcache->depot_size; ++j) {
    > iova_magazine_free_pfns(rcache->depot[j], iovad);
    > diff --git a/drivers/vdpa/vdpa_user/iova_domain.c b/drivers/vdpa/vdpa_user/iova_domain.c
    > index 2b1143f11d8f..d2ffdbf5f29c 100644
    > --- a/drivers/vdpa/vdpa_user/iova_domain.c
    > +++ b/drivers/vdpa/vdpa_user/iova_domain.c
    > @@ -285,25 +285,28 @@ static int vduse_domain_init_bounce_map(struct vduse_iova_domain *domain)
    > }
    >
    > static dma_addr_t
    > -vduse_domain_alloc_iova(struct iova_domain *iovad,
    > +vduse_domain_alloc_iova(struct iova_caching_domain *rcached,
    > unsigned long size, unsigned long limit)
    > {
    > + struct iova_domain *iovad = &rcached->iovad;
    > unsigned long shift = iova_shift(iovad);
    > unsigned long iova_len = iova_align(iovad, size) >> shift;
    > unsigned long iova_pfn;
    >
    > - iova_pfn = alloc_iova_fast(iovad, iova_len, limit >> shift, true);
    > + iova_pfn = alloc_iova_fast(rcached, iova_len, limit >> shift, true);
    >
    > return iova_pfn << shift;
    > }
    >
    > -static void vduse_domain_free_iova(struct iova_domain *iovad,
    > +static void vduse_domain_free_iova(struct iova_caching_domain *rcached,
    > dma_addr_t iova, size_t size)
    > {
    > + struct iova_domain *iovad = &rcached->iovad;
    > +
    > unsigned long shift = iova_shift(iovad);
    > unsigned long iova_len = iova_align(iovad, size) >> shift;
    >
    > - free_iova_fast(iovad, iova >> shift, iova_len);
    > + free_iova_fast(rcached, iova >> shift, iova_len);
    > }
    >
    > dma_addr_t vduse_domain_map_page(struct vduse_iova_domain *domain,
    > @@ -311,10 +314,10 @@ dma_addr_t vduse_domain_map_page(struct vduse_iova_domain *domain,
    > size_t size, enum dma_data_direction dir,
    > unsigned long attrs)
    > {
    > - struct iova_domain *iovad = &domain->stream_iovad;
    > + struct iova_caching_domain *rcached = &domain->stream_iovad;
    > unsigned long limit = domain->bounce_size - 1;
    > phys_addr_t pa = page_to_phys(page) + offset;
    > - dma_addr_t iova = vduse_domain_alloc_iova(iovad, size, limit);
    > + dma_addr_t iova = vduse_domain_alloc_iova(rcached, size, limit);
    >
    > if (!iova)
    > return DMA_MAPPING_ERROR;
    > @@ -330,7 +333,7 @@ dma_addr_t vduse_domain_map_page(struct vduse_iova_domain *domain,
    >
    > return iova;
    > err:
    > - vduse_domain_free_iova(iovad, iova, size);
    > + vduse_domain_free_iova(rcached, iova, size);
    > return DMA_MAPPING_ERROR;
    > }
    >
    > @@ -338,22 +341,22 @@ void vduse_domain_unmap_page(struct vduse_iova_domain *domain,
    > dma_addr_t dma_addr, size_t size,
    > enum dma_data_direction dir, unsigned long attrs)
    > {
    > - struct iova_domain *iovad = &domain->stream_iovad;
    > + struct iova_caching_domain *rcached = &domain->stream_iovad;
    >
    > if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
    > vduse_domain_bounce(domain, dma_addr, size, DMA_FROM_DEVICE);
    >
    > vduse_domain_unmap_bounce_page(domain, (u64)dma_addr, (u64)size);
    > - vduse_domain_free_iova(iovad, dma_addr, size);
    > + vduse_domain_free_iova(rcached, dma_addr, size);
    > }
    >
    > void *vduse_domain_alloc_coherent(struct vduse_iova_domain *domain,
    > size_t size, dma_addr_t *dma_addr,
    > gfp_t flag, unsigned long attrs)
    > {
    > - struct iova_domain *iovad = &domain->consistent_iovad;
    > + struct iova_caching_domain *rcached = &domain->consistent_iovad;
    > unsigned long limit = domain->iova_limit;
    > - dma_addr_t iova = vduse_domain_alloc_iova(iovad, size, limit);
    > + dma_addr_t iova = vduse_domain_alloc_iova(rcached, size, limit);
    > void *orig = alloc_pages_exact(size, flag);
    >
    > if (!iova || !orig)
    > @@ -376,7 +379,7 @@ void *vduse_domain_alloc_coherent(struct vduse_iova_domain *domain,
    > if (orig)
    > free_pages_exact(orig, size);
    > if (iova)
    > - vduse_domain_free_iova(iovad, iova, size);
    > + vduse_domain_free_iova(rcached, iova, size);
    >
    > return NULL;
    > }
    > @@ -385,7 +388,7 @@ void vduse_domain_free_coherent(struct vduse_iova_domain *domain, size_t size,
    > void *vaddr, dma_addr_t dma_addr,
    > unsigned long attrs)
    > {
    > - struct iova_domain *iovad = &domain->consistent_iovad;
    > + struct iova_caching_domain *rcached = &domain->consistent_iovad;
    > struct vhost_iotlb_map *map;
    > struct vdpa_map_file *map_file;
    > phys_addr_t pa;
    > @@ -404,7 +407,7 @@ void vduse_domain_free_coherent(struct vduse_iova_domain *domain, size_t size,
    > vhost_iotlb_map_free(domain->iotlb, map);
    > spin_unlock(&domain->iotlb_lock);
    >
    > - vduse_domain_free_iova(iovad, dma_addr, size);
    > + vduse_domain_free_iova(rcached, dma_addr, size);
    > free_pages_exact(phys_to_virt(pa), size);
    > }
    >
    > @@ -453,8 +456,8 @@ static int vduse_domain_release(struct inode *inode, struct file *file)
    > vduse_iotlb_del_range(domain, 0, ULLONG_MAX);
    > vduse_domain_free_bounce_pages(domain);
    > spin_unlock(&domain->iotlb_lock);
    > - put_iova_domain(&domain->stream_iovad);
    > - put_iova_domain(&domain->consistent_iovad);
    > + put_iova_caching_domain(&domain->stream_iovad);
    > + put_iova_caching_domain(&domain->consistent_iovad);
    > vhost_iotlb_free(domain->iotlb);
    > vfree(domain->bounce_maps);
    > kfree(domain);
    > @@ -480,6 +483,7 @@ vduse_domain_create(unsigned long iova_limit, size_t bounce_size)
    > struct file *file;
    > struct vduse_bounce_map *map;
    > unsigned long pfn, bounce_pfns;
    > + int ret;
    >
    > bounce_pfns = PAGE_ALIGN(bounce_size) >> PAGE_SHIFT;
    > if (iova_limit <= bounce_size)
    > @@ -511,12 +515,21 @@ vduse_domain_create(unsigned long iova_limit, size_t bounce_size)
    >
    > domain->file = file;
    > spin_lock_init(&domain->iotlb_lock);
    > - init_iova_domain(&domain->stream_iovad,
    > - PAGE_SIZE, IOVA_START_PFN);
    > - init_iova_domain(&domain->consistent_iovad,
    > - PAGE_SIZE, bounce_pfns);
    > + ret = init_iova_caching_domain(&domain->stream_iovad,
    > + PAGE_SIZE, IOVA_START_PFN);
    > + if (ret)
    > + goto err_stream_domain;
    > + ret = init_iova_caching_domain(&domain->consistent_iovad,
    > + PAGE_SIZE, bounce_pfns);
    > + if (ret)
    > + goto err_consistent_domain;
    >
    > return domain;
    > +
    > +err_consistent_domain:
    > + put_iova_caching_domain(&domain->stream_iovad);
    > +err_stream_domain:
    > + fput(domain->file);
    > err_file:
    > vfree(domain->bounce_maps);
    > err_map:
    > diff --git a/drivers/vdpa/vdpa_user/iova_domain.h b/drivers/vdpa/vdpa_user/iova_domain.h
    > index 2722d9b8e21a..38576e1d3b2c 100644
    > --- a/drivers/vdpa/vdpa_user/iova_domain.h
    > +++ b/drivers/vdpa/vdpa_user/iova_domain.h
    > @@ -25,8 +25,8 @@ struct vduse_bounce_map {
    > };
    >
    > struct vduse_iova_domain {
    > - struct iova_domain stream_iovad;
    > - struct iova_domain consistent_iovad;
    > + struct iova_caching_domain stream_iovad;
    > + struct iova_caching_domain consistent_iovad;
    > struct vduse_bounce_map *bounce_maps;
    > size_t bounce_size;
    > unsigned long iova_limit;
    > diff --git a/include/linux/iova.h b/include/linux/iova.h
    > index ef3b0f8f8a31..858ca7a5f1fa 100644
    > --- a/include/linux/iova.h
    > +++ b/include/linux/iova.h
    > @@ -48,6 +48,10 @@ struct iova_domain {
    > unsigned long dma_32bit_pfn;
    > unsigned long max32_alloc_size; /* Size of last failed allocation */
    > struct iova anchor; /* rbtree lookup anchor */
    > +};
    > +
    > +struct iova_caching_domain {
    > + struct iova_domain iovad;
    > struct iova_rcache rcaches[IOVA_RANGE_CACHE_MAX_SIZE]; /* IOVA range caches */
    > struct hlist_node cpuhp_dead;
    > };
    > @@ -96,16 +100,20 @@ void __free_iova(struct iova_domain *iovad, struct iova *iova);
    > struct iova *alloc_iova(struct iova_domain *iovad, unsigned long size,
    > unsigned long limit_pfn,
    > bool size_aligned);
    > -void free_iova_fast(struct iova_domain *iovad, unsigned long pfn,
    > +void free_iova_fast(struct iova_caching_domain *rcached, unsigned long pfn,
    > unsigned long size);
    > -unsigned long alloc_iova_fast(struct iova_domain *iovad, unsigned long size,
    > - unsigned long limit_pfn, bool flush_rcache);
    > +unsigned long alloc_iova_fast(struct iova_caching_domain *rcached,
    > + unsigned long size, unsigned long limit_pfn,
    > + bool flush_rcache);
    > struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo,
    > unsigned long pfn_hi);
    > void init_iova_domain(struct iova_domain *iovad, unsigned long granule,
    > unsigned long start_pfn);
    > struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn);
    > void put_iova_domain(struct iova_domain *iovad);
    > +void put_iova_caching_domain(struct iova_caching_domain *rcached);
    > +int init_iova_caching_domain(struct iova_caching_domain *rcached,
    > + unsigned long granule, unsigned long start_pfn);
    > #else
    > static inline int iova_cache_get(void)
    > {
    > @@ -132,13 +140,13 @@ static inline struct iova *alloc_iova(struct iova_domain *iovad,
    > return NULL;
    > }
    >
    > -static inline void free_iova_fast(struct iova_domain *iovad,
    > +static inline void free_iova_fast(struct iova_caching_domain *rcached,
    > unsigned long pfn,
    > unsigned long size)
    > {
    > }
    >
    > -static inline unsigned long alloc_iova_fast(struct iova_domain *iovad,
    > +static inline unsigned long alloc_iova_fast(struct iova_caching_domain *rcached,
    > unsigned long size,
    > unsigned long limit_pfn,
    > bool flush_rcache)
    >

    \
     
     \ /
      Last update: 2021-12-20 09:51    [W:2.685 / U:0.052 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site