From: Jason Gunthorpe Date: Tue, 8 Apr 2025 16:54:01 +0000 (-0300) Subject: iommu/pages: Move from struct page to struct ioptdesc and folio X-Git-Url: http://git.monstr.eu/?a=commitdiff_plain;h=212fcf36c6a6ee948790f9b860e89f87c68deb35;p=linux-2.6-microblaze.git iommu/pages: Move from struct page to struct ioptdesc and folio This brings the iommu page table allocator into the modern world of having its own private page descriptor and not re-using fields from struct page for its own purpose. It follows the basic pattern of struct ptdesc which did this transformation for the CPU page table allocator. Currently iommu-pages is pretty basic so this isn't a huge benefit, however I see a coming need for features that CPU allocator has, like sub PAGE_SIZE allocations, and RCU freeing. This provides the base infrastructure to implement those cleanly. Remove numa_node_id() calls from the inlines and instead use NUMA_NO_NODE which will get switched to numa_mem_id(), which seems to be the right ID to use for memory allocations. Reviewed-by: Lu Baolu Tested-by: Nicolin Chen Tested-by: Alejandro Jimenez Signed-off-by: Jason Gunthorpe Link: https://lore.kernel.org/r/13-v4-c8663abbb606+3f7-iommu_pages_jgg@nvidia.com Signed-off-by: Joerg Roedel --- diff --git a/drivers/iommu/iommu-pages.c b/drivers/iommu/iommu-pages.c index 6eacb6a34586..3077df642adb 100644 --- a/drivers/iommu/iommu-pages.c +++ b/drivers/iommu/iommu-pages.c @@ -7,6 +7,21 @@ #include #include +#define IOPTDESC_MATCH(pg_elm, elm) \ + static_assert(offsetof(struct page, pg_elm) == \ + offsetof(struct ioptdesc, elm)) +IOPTDESC_MATCH(flags, __page_flags); +IOPTDESC_MATCH(lru, iopt_freelist_elm); /* Ensure bit 0 is clear */ +IOPTDESC_MATCH(mapping, __page_mapping); +IOPTDESC_MATCH(private, _private); +IOPTDESC_MATCH(page_type, __page_type); +IOPTDESC_MATCH(_refcount, __page_refcount); +#ifdef CONFIG_MEMCG +IOPTDESC_MATCH(memcg_data, memcg_data); +#endif +#undef IOPTDESC_MATCH +static_assert(sizeof(struct ioptdesc) <= sizeof(struct page)); + /** * iommu_alloc_pages_node - Allocate a zeroed page of a given order from * specific NUMA node @@ -20,10 +35,17 @@ void *iommu_alloc_pages_node(int nid, gfp_t gfp, unsigned int order) { const unsigned long pgcnt = 1UL << order; - struct page *page; + struct folio *folio; - page = alloc_pages_node(nid, gfp | __GFP_ZERO | __GFP_COMP, order); - if (unlikely(!page)) + /* + * __folio_alloc_node() does not handle NUMA_NO_NODE like + * alloc_pages_node() did. + */ + if (nid == NUMA_NO_NODE) + nid = numa_mem_id(); + + folio = __folio_alloc_node(gfp | __GFP_ZERO, order, nid); + if (unlikely(!folio)) return NULL; /* @@ -35,21 +57,21 @@ void *iommu_alloc_pages_node(int nid, gfp_t gfp, unsigned int order) * This is necessary for the proper accounting as IOMMU state can be * rather large, i.e. multiple gigabytes in size. */ - mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, pgcnt); - mod_lruvec_page_state(page, NR_SECONDARY_PAGETABLE, pgcnt); + mod_node_page_state(folio_pgdat(folio), NR_IOMMU_PAGES, pgcnt); + lruvec_stat_mod_folio(folio, NR_SECONDARY_PAGETABLE, pgcnt); - return page_address(page); + return folio_address(folio); } EXPORT_SYMBOL_GPL(iommu_alloc_pages_node); -static void __iommu_free_page(struct page *page) +static void __iommu_free_desc(struct ioptdesc *iopt) { - unsigned int order = folio_order(page_folio(page)); - const unsigned long pgcnt = 1UL << order; + struct folio *folio = ioptdesc_folio(iopt); + const unsigned long pgcnt = 1UL << folio_order(folio); - mod_node_page_state(page_pgdat(page), NR_IOMMU_PAGES, -pgcnt); - mod_lruvec_page_state(page, NR_SECONDARY_PAGETABLE, -pgcnt); - put_page(page); + mod_node_page_state(folio_pgdat(folio), NR_IOMMU_PAGES, -pgcnt); + lruvec_stat_mod_folio(folio, NR_SECONDARY_PAGETABLE, -pgcnt); + folio_put(folio); } /** @@ -62,7 +84,7 @@ void iommu_free_pages(void *virt) { if (!virt) return; - __iommu_free_page(virt_to_page(virt)); + __iommu_free_desc(virt_to_ioptdesc(virt)); } EXPORT_SYMBOL_GPL(iommu_free_pages); @@ -74,9 +96,9 @@ EXPORT_SYMBOL_GPL(iommu_free_pages); */ void iommu_put_pages_list(struct iommu_pages_list *list) { - struct page *p, *tmp; + struct ioptdesc *iopt, *tmp; - list_for_each_entry_safe(p, tmp, &list->pages, lru) - __iommu_free_page(p); + list_for_each_entry_safe(iopt, tmp, &list->pages, iopt_freelist_elm) + __iommu_free_desc(iopt); } EXPORT_SYMBOL_GPL(iommu_put_pages_list); diff --git a/drivers/iommu/iommu-pages.h b/drivers/iommu/iommu-pages.h index 8dc0202bf108..f4578f252e25 100644 --- a/drivers/iommu/iommu-pages.h +++ b/drivers/iommu/iommu-pages.h @@ -9,6 +9,43 @@ #include +/** + * struct ioptdesc - Memory descriptor for IOMMU page tables + * @iopt_freelist_elm: List element for a struct iommu_pages_list + * + * This struct overlays struct page for now. Do not modify without a good + * understanding of the issues. + */ +struct ioptdesc { + unsigned long __page_flags; + + struct list_head iopt_freelist_elm; + unsigned long __page_mapping; + pgoff_t __index; + void *_private; + + unsigned int __page_type; + atomic_t __page_refcount; +#ifdef CONFIG_MEMCG + unsigned long memcg_data; +#endif +}; + +static inline struct ioptdesc *folio_ioptdesc(struct folio *folio) +{ + return (struct ioptdesc *)folio; +} + +static inline struct folio *ioptdesc_folio(struct ioptdesc *iopt) +{ + return (struct folio *)iopt; +} + +static inline struct ioptdesc *virt_to_ioptdesc(void *virt) +{ + return folio_ioptdesc(virt_to_folio(virt)); +} + void *iommu_alloc_pages_node(int nid, gfp_t gfp, unsigned int order); void iommu_free_pages(void *virt); void iommu_put_pages_list(struct iommu_pages_list *list); @@ -21,7 +58,7 @@ void iommu_put_pages_list(struct iommu_pages_list *list); static inline void iommu_pages_list_add(struct iommu_pages_list *list, void *virt) { - list_add_tail(&virt_to_page(virt)->lru, &list->pages); + list_add_tail(&virt_to_ioptdesc(virt)->iopt_freelist_elm, &list->pages); } /** @@ -56,7 +93,7 @@ static inline bool iommu_pages_list_empty(struct iommu_pages_list *list) */ static inline void *iommu_alloc_pages(gfp_t gfp, int order) { - return iommu_alloc_pages_node(numa_node_id(), gfp, order); + return iommu_alloc_pages_node(NUMA_NO_NODE, gfp, order); } /** @@ -79,7 +116,7 @@ static inline void *iommu_alloc_page_node(int nid, gfp_t gfp) */ static inline void *iommu_alloc_page(gfp_t gfp) { - return iommu_alloc_pages_node(numa_node_id(), gfp, 0); + return iommu_alloc_pages_node(NUMA_NO_NODE, gfp, 0); } #endif /* __IOMMU_PAGES_H */