Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417/* SPDX-License-Identifier: GPL-2.0 * * Network memory * * Author: Mina Almasry <almasrymina@google.com> */
#ifndef _NET_NETMEM_H#define _NET_NETMEM_H
#include <linux/dma-mapping.h>#include <linux/mm.h>#include <net/net_debug.h>
/* These fields in struct page are used by the page_pool and net stack: * * struct { * unsigned long pp_magic; * struct page_pool *pp; * unsigned long _pp_mapping_pad; * unsigned long dma_addr; * atomic_long_t pp_ref_count; * }; * * We mirror the page_pool fields here so the page_pool can access these * fields without worrying whether the underlying fields belong to a * page or netmem_desc. * * CAUTION: Do not update the fields in netmem_desc without also * updating the anonymous aliasing union in struct net_iov. */struct netmem_desc { unsigned long _flags; unsigned long pp_magic; struct page_pool *pp; unsigned long _pp_mapping_pad; unsigned long dma_addr; atomic_long_t pp_ref_count;};
#define NETMEM_DESC_ASSERT_OFFSET(pg, desc) \ static_assert(offsetof(struct page, pg) == \ offsetof(struct netmem_desc, desc))NETMEM_DESC_ASSERT_OFFSET(flags, _flags);NETMEM_DESC_ASSERT_OFFSET(pp_magic, pp_magic);NETMEM_DESC_ASSERT_OFFSET(pp, pp);NETMEM_DESC_ASSERT_OFFSET(_pp_mapping_pad, _pp_mapping_pad);NETMEM_DESC_ASSERT_OFFSET(dma_addr, dma_addr);NETMEM_DESC_ASSERT_OFFSET(pp_ref_count, pp_ref_count);#undef NETMEM_DESC_ASSERT_OFFSET
/* * Since struct netmem_desc uses the space in struct page, the size * should be checked, until struct netmem_desc has its own instance from * slab, to avoid conflicting with other members within struct page. */static_assert(sizeof(struct netmem_desc) <= offsetof(struct page, _refcount));
/* net_iov */
DECLARE_STATIC_KEY_FALSE(page_pool_mem_providers);
/* We overload the LSB of the struct page pointer to indicate whether it's * a page or net_iov. */#define NET_IOV 0x01UL
enum net_iov_type { NET_IOV_DMABUF, NET_IOV_IOURING,};
/* A memory descriptor representing abstract networking I/O vectors, * generally for non-pages memory that doesn't have its corresponding * struct page and needs to be explicitly allocated through slab. * * net_iovs are allocated and used by networking code, and the size of * the chunk is PAGE_SIZE. * * This memory can be any form of non-struct paged memory. Examples * include imported dmabuf memory and imported io_uring memory. See * net_iov_type for all the supported types. * * @pp_magic: pp field, similar to the one in struct page/struct * netmem_desc. * @pp: the pp this net_iov belongs to, if any. * @dma_addr: the dma addrs of the net_iov. Needed for the network * card to send/receive this net_iov. * @pp_ref_count: the pp ref count of this net_iov, exactly the same * usage as struct page/struct netmem_desc. * @owner: the net_iov_area this net_iov belongs to, if any. * @type: the type of the memory. Different types of net_iovs are * supported. */struct net_iov { struct netmem_desc desc; enum net_iov_type type; struct net_iov_area *owner;};
struct net_iov_area { /* Array of net_iovs for this area. */ struct net_iov *niovs; size_t num_niovs;
/* Offset into the dma-buf where this chunk starts. */ unsigned long base_virtual;};
static inline struct net_iov_area *net_iov_owner(const struct net_iov *niov){ return niov->owner;}
static inline unsigned int net_iov_idx(const struct net_iov *niov){ return niov - net_iov_owner(niov)->niovs;}
/* Initialize a niov: stamp the owning area, the memory provider type. */static inline void net_iov_init(struct net_iov *niov, struct net_iov_area *owner, enum net_iov_type type){ niov->owner = owner; niov->type = type;}
/* netmem */
/** * typedef netmem_ref - a nonexistent type marking a reference to generic * network memory. * * A netmem_ref can be a struct page* or a struct net_iov* underneath. * * Use the supplied helpers to obtain the underlying memory pointer and fields. */typedef unsigned long __bitwise netmem_ref;
static inline bool netmem_is_net_iov(const netmem_ref netmem){ return (__force unsigned long)netmem & NET_IOV;}
/** * __netmem_to_page - unsafely get pointer to the &page backing @netmem * @netmem: netmem reference to convert * * Unsafe version of netmem_to_page(). When @netmem is always page-backed, * e.g. when it's a header buffer, performs faster and generates smaller * object code (no check for the LSB, no WARN). When @netmem points to IOV, * provokes undefined behaviour. * * Return: pointer to the &page (garbage if @netmem is not page-backed). */static inline struct page *__netmem_to_page(netmem_ref netmem){ return (__force struct page *)netmem;}
static inline struct page *netmem_to_page(netmem_ref netmem){ if (WARN_ON_ONCE(netmem_is_net_iov(netmem))) return NULL;
return __netmem_to_page(netmem);}
static inline struct net_iov *netmem_to_net_iov(netmem_ref netmem){ if (netmem_is_net_iov(netmem)) return (struct net_iov *)((__force unsigned long)netmem & ~NET_IOV);
DEBUG_NET_WARN_ON_ONCE(true); return NULL;}
static inline netmem_ref net_iov_to_netmem(struct net_iov *niov){ return (__force netmem_ref)((unsigned long)niov | NET_IOV);}
#define page_to_netmem(p) (_Generic((p), \ const struct page * : (__force const netmem_ref)(p), \ struct page * : (__force netmem_ref)(p)))
/** * virt_to_netmem - convert virtual memory pointer to a netmem reference * @data: host memory pointer to convert * * Return: netmem reference to the &page backing this virtual address. */static inline netmem_ref virt_to_netmem(const void *data){ return page_to_netmem(virt_to_page(data));}
static inline int netmem_ref_count(netmem_ref netmem){ /* The non-pp refcount of net_iov is always 1. On net_iov, we only * support pp refcounting which uses the pp_ref_count field. */ if (netmem_is_net_iov(netmem)) return 1;
return page_ref_count(netmem_to_page(netmem));}
static inline unsigned long netmem_pfn_trace(netmem_ref netmem){ if (netmem_is_net_iov(netmem)) return 0;
return page_to_pfn(netmem_to_page(netmem));}
/* XXX: How to extract netmem_desc from page must be changed, once * netmem_desc no longer overlays on page and will be allocated through * slab. */#define __pp_page_to_nmdesc(p) (_Generic((p), \ const struct page * : (const struct netmem_desc *)(p), \ struct page * : (struct netmem_desc *)(p)))
/* CAUTION: Check if the page is a pp page before calling this helper or * know it's a pp page. */#define pp_page_to_nmdesc(p) \({ \ DEBUG_NET_WARN_ON_ONCE(!page_pool_page_is_pp(p)); \ __pp_page_to_nmdesc(p); \})
/** * __netmem_to_nmdesc - unsafely get pointer to the &netmem_desc backing * @netmem * @netmem: netmem reference to convert * * Unsafe version that can be used only when @netmem is always backed by * system memory, performs faster and generates smaller object code (no * check for the LSB, no WARN). When @netmem points to IOV, provokes * undefined behaviour. * * Return: pointer to the &netmem_desc (garbage if @netmem is not backed * by system memory). */static inline struct netmem_desc *__netmem_to_nmdesc(netmem_ref netmem){ return (__force struct netmem_desc *)netmem;}
/* netmem_to_nmdesc - convert netmem_ref to struct netmem_desc * for * access to common fields. * @netmem: netmem reference to get netmem_desc. * * All the sub types of netmem_ref (netmem_desc, net_iov) have the same * pp, pp_magic, dma_addr, and pp_ref_count fields via netmem_desc. * * Return: the pointer to struct netmem_desc * regardless of its * underlying type. */static inline struct netmem_desc *netmem_to_nmdesc(netmem_ref netmem){ void *p = (void *)((__force unsigned long)netmem & ~NET_IOV);
if (netmem_is_net_iov(netmem)) return &((struct net_iov *)p)->desc;
return __pp_page_to_nmdesc((struct page *)p);}
/** * __netmem_get_pp - unsafely get pointer to the &page_pool backing @netmem * @netmem: netmem reference to get the pointer from * * Unsafe version of netmem_get_pp(). When @netmem is always page-backed, * e.g. when it's a header buffer, performs faster and generates smaller * object code (avoids clearing the LSB). When @netmem points to IOV, * provokes invalid memory access. * * Return: pointer to the &page_pool (garbage if @netmem is not page-backed). */static inline struct page_pool *__netmem_get_pp(netmem_ref netmem){ return __netmem_to_nmdesc(netmem)->pp;}
static inline struct page_pool *netmem_get_pp(netmem_ref netmem){ return netmem_to_nmdesc(netmem)->pp;}
static inline atomic_long_t *netmem_get_pp_ref_count_ref(netmem_ref netmem){ return &netmem_to_nmdesc(netmem)->pp_ref_count;}
static inline bool netmem_is_pref_nid(netmem_ref netmem, int pref_nid){ /* NUMA node preference only makes sense if we're allocating * system memory. Memory providers (which give us net_iovs) * choose for us. */ if (netmem_is_net_iov(netmem)) return true;
return page_to_nid(netmem_to_page(netmem)) == pref_nid;}
static inline netmem_ref netmem_compound_head(netmem_ref netmem){ /* niov are never compounded */ if (netmem_is_net_iov(netmem)) return netmem;
return page_to_netmem(compound_head(netmem_to_page(netmem)));}
/** * __netmem_address - unsafely get pointer to the memory backing @netmem * @netmem: netmem reference to get the pointer for * * Unsafe version of netmem_address(). When @netmem is always page-backed, * e.g. when it's a header buffer, performs faster and generates smaller * object code (no check for the LSB). When @netmem points to IOV, provokes * undefined behaviour. * * Return: pointer to the memory (garbage if @netmem is not page-backed). */static inline void *__netmem_address(netmem_ref netmem){ return page_address(__netmem_to_page(netmem));}
static inline void *netmem_address(netmem_ref netmem){ if (netmem_is_net_iov(netmem)) return NULL;
return __netmem_address(netmem);}
/** * netmem_is_pfmemalloc - check if @netmem was allocated under memory pressure * @netmem: netmem reference to check * * Return: true if @netmem is page-backed and the page was allocated under * memory pressure, false otherwise. */static inline bool netmem_is_pfmemalloc(netmem_ref netmem){ if (netmem_is_net_iov(netmem)) return false;
return page_is_pfmemalloc(netmem_to_page(netmem));}
static inline unsigned long netmem_get_dma_addr(netmem_ref netmem){ return netmem_to_nmdesc(netmem)->dma_addr;}
#if defined(CONFIG_NET_DEVMEM)static inline bool net_is_devmem_iov(const struct net_iov *niov){ return niov->type == NET_IOV_DMABUF;}#elsestatic inline bool net_is_devmem_iov(const struct net_iov *niov){ return false;}#endif
void __get_netmem(netmem_ref netmem);void __put_netmem(netmem_ref netmem);
static __always_inline void get_netmem(netmem_ref netmem){ if (netmem_is_net_iov(netmem)) __get_netmem(netmem); else get_page(netmem_to_page(netmem));}
static __always_inline void put_netmem(netmem_ref netmem){ if (netmem_is_net_iov(netmem)) __put_netmem(netmem); else put_page(netmem_to_page(netmem));}
#define netmem_dma_unmap_addr_set(NETMEM, PTR, ADDR_NAME, VAL) \ do { \ if (!netmem_is_net_iov(NETMEM)) \ dma_unmap_addr_set(PTR, ADDR_NAME, VAL); \ else \ dma_unmap_addr_set(PTR, ADDR_NAME, 0); \ } while (0)
static inline void netmem_dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size, enum dma_data_direction dir, unsigned long attrs){ if (!addr) return;
dma_unmap_page_attrs(dev, addr, size, dir, attrs);}
#endif /* _NET_NETMEM_H */