Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437/* SPDX-License-Identifier: GPL-2.0-only *//* Copyright (C) 2025 Intel Corporation */
#ifndef __LIBIE_CONTROLQ_H#define __LIBIE_CONTROLQ_H
#include <linux/dmapool.h>#include <net/libeth/rx.h>
#include <linux/net/intel/libie/pci.h>#include <linux/net/intel/virtchnl2.h>
/* Default mailbox control queue */#define LIBIE_CTLQ_MBX_ID -1#define LIBIE_CTLQ_MAX_BUF_LEN SZ_4K
/** * enum libie_ctlq_type - control queue type * @LIBIE_CTLQ_TYPE_TX: basic Tx control queue * @LIBIE_CTLQ_TYPE_RX: basic Rx control queue */enum libie_ctlq_type { LIBIE_CTLQ_TYPE_TX = 0, LIBIE_CTLQ_TYPE_RX = 1,};
/* Opcode used to send controlq message to the control plane */#define LIBIE_CTLQ_SEND_MSG_TO_CP 0x801#define LIBIE_CTLQ_SEND_MSG_TO_PEER 0x804
#define LIBIE_CP_TX_COPYBREAK 128
/** * struct libie_ctlq_ctx - contains controlq info and MMIO region info * @mmio_info: MMIO region info structure * @ctlqs: list that stores all the control queues * @ctlqs_lock: lock for control queue list */struct libie_ctlq_ctx { struct libie_mmio_info mmio_info; struct list_head ctlqs; spinlock_t ctlqs_lock; /* protects the ctlqs list */};
/** * struct libie_ctlq_reg - structure representing virtual addresses of the * controlq registers and masks * @head: controlq head register address * @tail: controlq tail register address * @len: register address to write controlq length and enable bit * @addr_high: register address to write the upper 32b of ring physical address * @addr_low: register address to write the lower 32b of ring physical address * @len_mask: mask to read the controlq length * @len_ena_mask: mask to write the controlq enable bit * @head_mask: mask to read the head value */struct libie_ctlq_reg { void __iomem *head; void __iomem *tail; void __iomem *len; void __iomem *addr_high; void __iomem *addr_low; u32 len_mask; u32 len_ena_mask; u32 head_mask;};
/** * struct libie_cp_dma_mem - structure for DMA memory * @va: virtual address * @pa: physical address * @size: memory size * @direction: memory to device or device to memory */struct libie_cp_dma_mem { void *va; dma_addr_t pa; size_t size; int direction;};
/** * struct libie_ctlq_msg - control queue message data * @flags: refer to 'Flags sub-structure' definitions * @opcode: infrastructure message opcode * @data_len: size of the payload * @func_id: queue id for mailbox selection, 0 for default mailbox (Tx) * @hw_retval: execution status from the HW (Rx) * @chnl_opcode: virtchnl message opcode * @chnl_retval: virtchnl return value * @param0: indirect message raw parameter0 * @sw_cookie: used to verify the response of the sent virtchnl message * @virt_flags: virtchnl capability flags * @addr_param: additional parameters in place of the address, given no buffer * @recv_mem: virtual address and size of the buffer that contains * the indirect response * @send_mem: physical and virtual address of the DMA buffer, * used for sending */struct libie_ctlq_msg { u16 flags; u16 opcode; u16 data_len; union { u16 func_id; u16 hw_retval; }; u32 chnl_opcode; u32 chnl_retval; u32 param0; u16 sw_cookie; u16 virt_flags; u64 addr_param; union { struct kvec recv_mem; struct libie_cp_dma_mem send_mem; };};
/** * struct libie_ctlq_create_info - control queue create information * @type: control queue type (Rx or Tx) * @id: queue offset passed as input, -1 for default mailbox * @reg: registers accessed by control queue * @len: controlq length */struct libie_ctlq_create_info { enum libie_ctlq_type type; int id; struct libie_ctlq_reg reg; u16 len;};
/** * struct libie_ctlq_info - control queue information * @list: used to add a controlq to the list of queues in libie_ctlq_ctx * @type: control queue type * @qid: queue identifier * @lock: control queue lock * @ring_mem: descriptor ring DMA memory * @descs: array of descriptors * @rx_fqes: array of controlq Rx buffers * @tx_msg: Tx messages sent to hardware * @reg: registers used by control queue * @dev: device that owns this control queue * @pp: page pool for controlq Rx buffers * @truesize: size to allocate per buffer * @next_to_clean: next descriptor to be cleaned * @next_to_use: next available slot to send buffer (Tx queue) * @next_to_post: next available slot to post buffers to (Rx queue) * @ring_len: length of the descriptor ring */struct libie_ctlq_info { struct list_head list; enum libie_ctlq_type type; int qid; spinlock_t lock; /* for concurrent processing */ struct libie_cp_dma_mem ring_mem; struct libie_ctlq_desc *descs; union { struct libeth_fqe *rx_fqes; struct libie_ctlq_msg **tx_msg; }; struct libie_ctlq_reg reg; struct device *dev; struct page_pool *pp; u32 truesize; u32 next_to_clean; union { u32 next_to_use; u32 next_to_post; }; u32 ring_len;};
#define LIBIE_CTLQ_MBX_ATQ_LEN GENMASK(9, 0)
/* libie controlq descriptor qword0 details */
/* Flags sub-structure * |0 |1 |2 |3 |4 |5 |6 |7 |8 |9 |10 |11 |12 |13 |14 |15 | * |DD |CMP|ERR| * RSV * |FTYPE | *RSV* |RD |VFC|BUF| HOST_ID | */#define LIBIE_CTLQ_DESC_FLAG_DD BIT(0)#define LIBIE_CTLQ_DESC_FLAG_CMP BIT(1)#define LIBIE_CTLQ_DESC_FLAG_ERR BIT(2)#define LIBIE_CTLQ_DESC_FLAG_FTYPE_VM BIT(6)#define LIBIE_CTLQ_DESC_FLAG_FTYPE_PF BIT(7)#define LIBIE_CTLQ_DESC_FLAG_FTYPE GENMASK(7, 6)#define LIBIE_CTLQ_DESC_FLAG_RD BIT(10)#define LIBIE_CTLQ_DESC_FLAG_VFC BIT(11)#define LIBIE_CTLQ_DESC_FLAG_BUF BIT(12)#define LIBIE_CTLQ_DESC_FLAG_HOST_ID GENMASK(15, 13)
#define LIBIE_CTLQ_DESC_FLAGS GENMASK(15, 0)#define LIBIE_CTLQ_DESC_INFRA_OPCODE GENMASK_ULL(31, 16)#define LIBIE_CTLQ_DESC_DATA_LEN GENMASK_ULL(47, 32)#define LIBIE_CTLQ_DESC_HW_RETVAL GENMASK_ULL(63, 48)
#define LIBIE_CTLQ_DESC_PFID_VFID GENMASK_ULL(63, 48)
/* libie controlq descriptor qword1 details */#define LIBIE_CTLQ_DESC_VIRTCHNL_OPCODE GENMASK(27, 0)#define LIBIE_CTLQ_DESC_VIRTCHNL_DESC_TYPE GENMASK_ULL(31, 28)#define LIBIE_CTLQ_DESC_VIRTCHNL_MSG_RET_VAL GENMASK_ULL(63, 32)
/* libie controlq descriptor qword2 details */#define LIBIE_CTLQ_DESC_MSG_PARAM0 GENMASK_ULL(31, 0)#define LIBIE_CTLQ_DESC_SW_COOKIE GENMASK_ULL(47, 32)#define LIBIE_CTLQ_DESC_VIRTCHNL_FLAGS GENMASK_ULL(63, 48)
/* libie controlq descriptor qword3 details */#define LIBIE_CTLQ_DESC_DATA_ADDR_HIGH GENMASK_ULL(31, 0)#define LIBIE_CTLQ_DESC_DATA_ADDR_LOW GENMASK_ULL(63, 32)
/** * struct libie_ctlq_desc - control queue descriptor format * @qword0: flags, message opcode, data length etc * @qword1: virtchnl opcode, descriptor type and return value * @qword2: indirect message parameters * @qword3: indirect message buffer address */struct libie_ctlq_desc { __le64 qword0; __le64 qword1; __le64 qword2; __le64 qword3;};
/** * struct libie_ctlq_clean_params - cleaning parameters for Tx messages * @rel_dma_mem: non-sleeping callback to put the DMA buffer after send * @rel_ctx: additional context for release callback * @ctlq: control queue information * @num_msgs: number of messages to be cleaned * @force: clean even if DD is not yet set, use only for final cleanup */struct libie_ctlq_clean_params { void (*rel_dma_mem)(const void *ctx, struct libie_cp_dma_mem *dma_mem); const void *rel_ctx; struct libie_ctlq_info *ctlq; u16 num_msgs; bool force;};
/** * libie_ctlq_release_rx_buf - Release Rx buffer for a specific control queue * @rx_buf: Rx buffer to be freed * * Driver uses this function to post back the Rx buffer after the usage. */static inline void libie_ctlq_release_rx_buf(struct kvec *rx_buf){ netmem_ref netmem;
if (!rx_buf->iov_base) return;
netmem = virt_to_netmem(rx_buf->iov_base); page_pool_put_full_netmem(netmem_get_pp(netmem), netmem, false);}
int libie_ctlq_init(struct libie_ctlq_ctx *ctx, const struct libie_ctlq_create_info *qinfo, u32 numq);void libie_ctlq_deinit(struct libie_ctlq_ctx *ctx);
struct libie_ctlq_info *libie_find_ctlq(struct libie_ctlq_ctx *ctx, enum libie_ctlq_type type, int id);
u32 libie_ctlq_send_desc_avail(const struct libie_ctlq_info *ctlq);void libie_ctlq_send(struct libie_ctlq_info *ctlq, u32 num_q_msg);u32 libie_ctlq_send_clean(const struct libie_ctlq_clean_params *params);u32 libie_ctlq_recv(struct libie_ctlq_info *ctlq, struct libie_ctlq_msg *msg, u32 num_q_msg);
int libie_ctlq_post_rx_buffs(struct libie_ctlq_info *ctlq);
/* Only 8 bits are available in descriptor for Xn index */#define LIBIE_CTLQ_MAX_XN_ENTRIES 256#define LIBIE_CTLQ_XN_COOKIE_M GENMASK(15, 8)#define LIBIE_CTLQ_XN_INDEX_M GENMASK(7, 0)
/** * enum libie_ctlq_xn_state - Transaction state of a virtchnl message * @LIBIE_CTLQ_XN_IDLE: transaction is available to use * @LIBIE_CTLQ_XN_WAITING: waiting for transaction to complete * @LIBIE_CTLQ_XN_COMPLETED_SUCCESS: transaction completed with success * @LIBIE_CTLQ_XN_COMPLETED_FAILED: transaction completed with failure * @LIBIE_CTLQ_XN_ASYNC: asynchronous virtchnl message transaction type * @LIBIE_CTLQ_XN_SHUTDOWN: transaction cannot be used anymore */enum libie_ctlq_xn_state { LIBIE_CTLQ_XN_IDLE = 0, LIBIE_CTLQ_XN_WAITING, LIBIE_CTLQ_XN_COMPLETED_SUCCESS, LIBIE_CTLQ_XN_COMPLETED_FAILED, LIBIE_CTLQ_XN_ASYNC, LIBIE_CTLQ_XN_SHUTDOWN,};
/** * struct libie_ctlq_xn - structure representing a virtchnl transaction entry * @resp_cb: non-sleeping callback to handle the response to an async message * @xn_lock: lock to protect the transaction entry state * @cmd_completion_event: wait until reply is received or xn is terminated * @small_dma_mem: DMA memory for copying small send buffers from stack, * is recycled when response is received or on timeout * @send_dma_mem: DMA memory of send buffer * @recv_mem: receive buffer * @send_ctx: context for callback function * @timeout_ms: Xn transaction timeout in msecs * @timestamp: timestamp to record the Xn send * @tx_msg: control queue Tx message slot to track small DMA usage * @virtchnl_opcode: virtchnl command opcode used for Xn transaction * @state: transaction state of a virtchnl message * @cookie: unique message identifier, incremented every time the slot is used * @index: index of the transaction entry */struct libie_ctlq_xn { void (*resp_cb)(void *ctx, struct kvec *mem, int status); spinlock_t xn_lock; /* protects state */ struct completion cmd_completion_event; struct libie_cp_dma_mem small_dma_mem; struct libie_cp_dma_mem send_dma_mem; struct kvec recv_mem; void *send_ctx; u64 timeout_ms; ktime_t timestamp; struct libie_ctlq_msg *tx_msg; u32 virtchnl_opcode; enum libie_ctlq_xn_state state; u8 cookie; u8 index;};
/** * struct libie_ctlq_xn_manager - structure representing the array of virtchnl * transaction entries * @ctx: pointer to controlq context structure * @free_xns_bm_lock: lock to protect the free Xn entries bit map * @free_xns_bm: bitmap that represents the free Xn entries * @ring: array of Xn entries * @small_buff_pool: DMA pool for small send buffers * @can_destroy: completion, triggered by the last released transaction * @shutdown: shutdown process has been started, no new transactions allowed */struct libie_ctlq_xn_manager { struct libie_ctlq_ctx *ctx; spinlock_t free_xns_bm_lock; /* get/check entries */ DECLARE_BITMAP(free_xns_bm, LIBIE_CTLQ_MAX_XN_ENTRIES); struct libie_ctlq_xn ring[LIBIE_CTLQ_MAX_XN_ENTRIES]; struct dma_pool *small_buff_pool; struct completion can_destroy; bool shutdown;};
/** * struct libie_ctlq_xn_send_params - structure representing send Xn entry * @resp_cb: non-sleeping callback to handle the response to an async message * @rel_tx_buf: non-sleeping callback for freeing the send buffer * @xnm: Xn manager to process Xn entries * @ctlq: send control queue information * @ctlq_msg: control queue message information * @send_buf: buffer that carries outgoing message data, buffers larger than * LIBIE_CP_TX_COPYBREAK bytes will always be consumed * @recv_mem: receive buffer * @send_ctx: context for callback function * @timeout_ms: virtchnl transaction timeout in msecs * @chnl_opcode: virtchnl message opcode */struct libie_ctlq_xn_send_params { void (*resp_cb)(void *ctx, struct kvec *mem, int status); void (*rel_tx_buf)(const void *buf_va); struct libie_ctlq_xn_manager *xnm; struct libie_ctlq_info *ctlq; struct libie_ctlq_msg *ctlq_msg; struct kvec send_buf; struct kvec recv_mem; void *send_ctx; u64 timeout_ms; u32 chnl_opcode;};
/** * libie_cp_can_send_onstack - can a message be sent using a stack variable * @size: ctlq data buffer size * * Return: %true if the message size is small enough for caller to pass * an on-stack buffer, %false if kmalloc is needed */static inline bool libie_cp_can_send_onstack(u32 size){ return size <= LIBIE_CP_TX_COPYBREAK;}
/** * struct libie_ctlq_xn_recv_params - request to receive xn responses * @ctlq_msg_handler: handler for Rx messages with no matching xn (mandatory) * @xnm: Xn manager to process Xn entries * @ctlq: control queue information * @budget: maximum number of messages to process */struct libie_ctlq_xn_recv_params { void (*ctlq_msg_handler)(struct libie_ctlq_ctx *ctx, struct libie_ctlq_msg *msg); struct libie_ctlq_xn_manager *xnm; struct libie_ctlq_info *ctlq; u32 budget;};
/** * struct libie_ctlq_xn_init_params - xn transaction manager parameters * @cctlq_info: control queue information * @ctx: pointer to controlq context structure * @xnm: Xn manager to process Xn entries * @num_qs: number of control queues to be initialized */struct libie_ctlq_xn_init_params { struct libie_ctlq_create_info *cctlq_info; struct libie_ctlq_ctx *ctx; struct libie_ctlq_xn_manager *xnm; u32 num_qs;};
int libie_ctlq_xn_init(struct libie_ctlq_xn_init_params *params);void libie_ctlq_xn_deinit(struct libie_ctlq_xn_manager *xnm, struct libie_ctlq_ctx *ctx);void libie_ctlq_xn_shutdown(struct libie_ctlq_xn_manager *xnm);int libie_ctlq_xn_send(struct libie_ctlq_xn_send_params *params);u32 libie_ctlq_xn_recv(struct libie_ctlq_xn_recv_params *params);u32 libie_ctlq_xn_send_clean(struct libie_ctlq_info *ctlq, void (*rel_tx_buf)(const void *buf_va), bool force);
#endif /* __LIBIE_CONTROLQ_H */