dlan17/linux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

idpf: use generic functions to build xdp_buff and skb

Browse Source 
In preparation of XDP support, move from having skb as the main frame
container during the Rx polling to &xdp_buff.
This allows to use generic and libeth helpers for building an XDP
buffer and changes the logics: now we try to allocate an skb only
when we processed all the descriptors related to the frame.
Store &libeth_xdp_stash instead of the skb pointer on the Rx queue.
It's only 8 bytes wider, but contains everything we may need.

Signed-off-by: Alexander Lobakin <aleksander.lobakin@intel.com>
Tested-by: Ramu R <ramu.r@intel.com>
Signed-off-by: Tony Nguyen <anthony.l.nguyen@intel.com>
This commit is contained in:
Alexander Lobakin
2025-09-08 11:05:17 -07:00
committed by Tony Nguyen
parent 705457e721
commit a4d755d104
3 changed files with 90 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files
drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c
+47 -57
View File
@@ -1,8 +1,7 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (C) 2023 Intel Corporation */
#include <net/libeth/rx.h>
#include <net/libeth/tx.h>
#include <net/libeth/xdp.h>
#include "idpf.h"
@@ -834,7 +833,7 @@ static void idpf_rx_singleq_flex_hash(struct idpf_rx_queue *rx_q,
}
/**
* idpf_rx_singleq_process_skb_fields - Populate skb header fields from Rx
* __idpf_rx_singleq_process_skb_fields - Populate skb header fields from Rx
* descriptor
* @rx_q: Rx ring being processed
* @skb: pointer to current skb being populated
@@ -846,17 +845,14 @@ static void idpf_rx_singleq_flex_hash(struct idpf_rx_queue *rx_q,
* other fields within the skb.
*/
static void
idpf_rx_singleq_process_skb_fields(struct idpf_rx_queue *rx_q,
struct sk_buff *skb,
const union virtchnl2_rx_desc *rx_desc,
u16 ptype)
__idpf_rx_singleq_process_skb_fields(struct idpf_rx_queue *rx_q,
struct sk_buff *skb,
const union virtchnl2_rx_desc *rx_desc,
u16 ptype)
{
struct libeth_rx_pt decoded = rx_q->rx_ptype_lkup[ptype];
struct libeth_rx_csum csum_bits;
/* modifies the skb - consumes the enet header */
skb->protocol = eth_type_trans(skb, rx_q->xdp_rxq.dev);
/* Check if we're using base mode descriptor IDs */
if (rx_q->rxdids == VIRTCHNL2_RXDID_1_32B_BASE_M) {
idpf_rx_singleq_base_hash(rx_q, skb, rx_desc, decoded);
@@ -867,7 +863,6 @@ idpf_rx_singleq_process_skb_fields(struct idpf_rx_queue *rx_q,
}
idpf_rx_singleq_csum(rx_q, skb, csum_bits, decoded);
skb_record_rx_queue(skb, rx_q->idx);
}
/**
@@ -1003,6 +998,32 @@ idpf_rx_singleq_extract_fields(const struct idpf_rx_queue *rx_q,
idpf_rx_singleq_extract_flex_fields(rx_desc, fields);
}
static bool
idpf_rx_singleq_process_skb_fields(struct sk_buff *skb,
const struct libeth_xdp_buff *xdp,
struct libeth_rq_napi_stats *rs)
{
struct libeth_rqe_info fields;
struct idpf_rx_queue *rxq;
rxq = libeth_xdp_buff_to_rq(xdp, typeof(*rxq), xdp_rxq);
idpf_rx_singleq_extract_fields(rxq, xdp->desc, &fields);
__idpf_rx_singleq_process_skb_fields(rxq, skb, xdp->desc,
fields.ptype);
return true;
}
static void idpf_xdp_run_pass(struct libeth_xdp_buff *xdp,
struct napi_struct *napi,
struct libeth_rq_napi_stats *rs,
const union virtchnl2_rx_desc *desc)
{
libeth_xdp_run_pass(xdp, NULL, napi, rs, desc, NULL,
idpf_rx_singleq_process_skb_fields);
}
/**
* idpf_rx_singleq_clean - Reclaim resources after receive completes
* @rx_q: rx queue to clean
@@ -1012,14 +1033,15 @@ idpf_rx_singleq_extract_fields(const struct idpf_rx_queue *rx_q,
*/
static int idpf_rx_singleq_clean(struct idpf_rx_queue *rx_q, int budget)
{
unsigned int total_rx_bytes = 0, total_rx_pkts = 0;
struct sk_buff *skb = rx_q->skb;
struct libeth_rq_napi_stats rs = { };
u16 ntc = rx_q->next_to_clean;
LIBETH_XDP_ONSTACK_BUFF(xdp);
u16 cleaned_count = 0;
bool failure = false;
libeth_xdp_init_buff(xdp, &rx_q->xdp, &rx_q->xdp_rxq);
/* Process Rx packets bounded by budget */
while (likely(total_rx_pkts < (unsigned int)budget)) {
while (likely(rs.packets < budget)) {
struct libeth_rqe_info fields = { };
union virtchnl2_rx_desc *rx_desc;
struct idpf_rx_buf *rx_buf;
@@ -1046,73 +1068,41 @@ static int idpf_rx_singleq_clean(struct idpf_rx_queue *rx_q, int budget)
idpf_rx_singleq_extract_fields(rx_q, rx_desc, &fields);
rx_buf = &rx_q->rx_buf[ntc];
if (!libeth_rx_sync_for_cpu(rx_buf, fields.len))
goto skip_data;
if (skb)
idpf_rx_add_frag(rx_buf, skb, fields.len);
else
skb = idpf_rx_build_skb(rx_buf, fields.len);
/* exit if we failed to retrieve a buffer */
if (!skb)
break;
skip_data:
libeth_xdp_process_buff(xdp, rx_buf, fields.len);
rx_buf->netmem = 0;
IDPF_SINGLEQ_BUMP_RING_IDX(rx_q, ntc);
cleaned_count++;
/* skip if it is non EOP desc */
if (idpf_rx_singleq_is_non_eop(rx_desc) || unlikely(!skb))
if (idpf_rx_singleq_is_non_eop(rx_desc) ||
unlikely(!xdp->data))
continue;
#define IDPF_RXD_ERR_S FIELD_PREP(VIRTCHNL2_RX_BASE_DESC_QW1_ERROR_M, \
VIRTCHNL2_RX_BASE_DESC_ERROR_RXE_M)
if (unlikely(idpf_rx_singleq_test_staterr(rx_desc,
IDPF_RXD_ERR_S))) {
dev_kfree_skb_any(skb);
skb = NULL;
libeth_xdp_return_buff_slow(xdp);
continue;
}
/* pad skb if needed (to make valid ethernet frame) */
if (eth_skb_pad(skb)) {
skb = NULL;
continue;
}
/* probably a little skewed due to removing CRC */
total_rx_bytes += skb->len;
/* protocol */
idpf_rx_singleq_process_skb_fields(rx_q, skb, rx_desc,
fields.ptype);
/* send completed skb up the stack */
napi_gro_receive(rx_q->pp->p.napi, skb);
skb = NULL;
/* update budget accounting */
total_rx_pkts++;
idpf_xdp_run_pass(xdp, rx_q->pp->p.napi, &rs, rx_desc);
}
rx_q->skb = skb;
rx_q->next_to_clean = ntc;
libeth_xdp_save_buff(&rx_q->xdp, xdp);
page_pool_nid_changed(rx_q->pp, numa_mem_id());
if (cleaned_count)
failure = idpf_rx_singleq_buf_hw_alloc_all(rx_q, cleaned_count);
idpf_rx_singleq_buf_hw_alloc_all(rx_q, cleaned_count);
u64_stats_update_begin(&rx_q->stats_sync);
u64_stats_add(&rx_q->q_stats.packets, total_rx_pkts);
u64_stats_add(&rx_q->q_stats.bytes, total_rx_bytes);
u64_stats_add(&rx_q->q_stats.packets, rs.packets);
u64_stats_add(&rx_q->q_stats.bytes, rs.bytes);
u64_stats_update_end(&rx_q->stats_sync);
/* guarantee a trip back through this routine if there was a failure */
return failure ? budget : (int)total_rx_pkts;
return rs.packets;
}
/**
drivers/net/ethernet/intel/idpf/idpf_txrx.c
+36 -109
View File
@@ -422,10 +422,7 @@ static void idpf_rx_desc_rel(struct idpf_rx_queue *rxq, struct device *dev,
if (!rxq)
return;
if (rxq->skb) {
dev_kfree_skb_any(rxq->skb);
rxq->skb = NULL;
}
libeth_xdp_return_stash(&rxq->xdp);
if (!idpf_is_queue_model_split(model))
idpf_rx_buf_rel_all(rxq);
@@ -2998,7 +2995,7 @@ idpf_rx_hwtstamp(const struct idpf_rx_queue *rxq,
}
/**
* idpf_rx_process_skb_fields - Populate skb header fields from Rx descriptor
* __idpf_rx_process_skb_fields - Populate skb header fields from Rx descriptor
* @rxq: Rx descriptor ring packet is being transacted on
* @skb: pointer to current skb being populated
* @rx_desc: Receive descriptor
@@ -3008,8 +3005,8 @@ idpf_rx_hwtstamp(const struct idpf_rx_queue *rxq,
* other fields within the skb.
*/
static int
idpf_rx_process_skb_fields(struct idpf_rx_queue *rxq, struct sk_buff *skb,
const struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc)
__idpf_rx_process_skb_fields(struct idpf_rx_queue *rxq, struct sk_buff *skb,
const struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc)
{
struct libeth_rx_csum csum_bits;
struct libeth_rx_pt decoded;
@@ -3025,9 +3022,6 @@ idpf_rx_process_skb_fields(struct idpf_rx_queue *rxq, struct sk_buff *skb,
if (idpf_queue_has(PTP, rxq))
idpf_rx_hwtstamp(rxq, rx_desc, skb);
skb->protocol = eth_type_trans(skb, rxq->xdp_rxq.dev);
skb_record_rx_queue(skb, rxq->idx);
if (le16_get_bits(rx_desc->hdrlen_flags,
VIRTCHNL2_RX_FLEX_DESC_ADV_RSC_M))
return idpf_rx_rsc(rxq, skb, rx_desc, decoded);
@@ -3038,23 +3032,24 @@ idpf_rx_process_skb_fields(struct idpf_rx_queue *rxq, struct sk_buff *skb,
return 0;
}
/**
* idpf_rx_add_frag - Add contents of Rx buffer to sk_buff as a frag
* @rx_buf: buffer containing page to add
* @skb: sk_buff to place the data into
* @size: packet length from rx_desc
*
* This function will add the data contained in rx_buf->page to the skb.
* It will just attach the page as a frag to the skb.
* The function will then update the page offset.
*/
void idpf_rx_add_frag(struct idpf_rx_buf *rx_buf, struct sk_buff *skb,
unsigned int size)
static bool idpf_rx_process_skb_fields(struct sk_buff *skb,
const struct libeth_xdp_buff *xdp,
struct libeth_rq_napi_stats *rs)
{
u32 hr = netmem_get_pp(rx_buf->netmem)->p.offset;
struct idpf_rx_queue *rxq;
skb_add_rx_frag_netmem(skb, skb_shinfo(skb)->nr_frags, rx_buf->netmem,
rx_buf->offset + hr, size, rx_buf->truesize);
rxq = libeth_xdp_buff_to_rq(xdp, typeof(*rxq), xdp_rxq);
return !__idpf_rx_process_skb_fields(rxq, skb, xdp->desc);
}
static void
idpf_xdp_run_pass(struct libeth_xdp_buff *xdp, struct napi_struct *napi,
struct libeth_rq_napi_stats *ss,
const struct virtchnl2_rx_flex_desc_adv_nic_3 *desc)
{
libeth_xdp_run_pass(xdp, NULL, napi, ss, desc, NULL,
idpf_rx_process_skb_fields);
}
/**
@@ -3098,36 +3093,6 @@ static u32 idpf_rx_hsplit_wa(const struct libeth_fqe *hdr,
return copy;
}
/**
* idpf_rx_build_skb - Allocate skb and populate it from header buffer
* @buf: Rx buffer to pull data from
* @size: the length of the packet
*
* This function allocates an skb. It then populates it with the page data from
* the current receive descriptor, taking care to set up the skb correctly.
*/
struct sk_buff *idpf_rx_build_skb(const struct libeth_fqe *buf, u32 size)
{
struct page *buf_page = __netmem_to_page(buf->netmem);
u32 hr = pp_page_to_nmdesc(buf_page)->pp->p.offset;
struct sk_buff *skb;
void *va;
va = page_address(buf_page) + buf->offset;
prefetch(va + hr);
skb = napi_build_skb(va, buf->truesize);
if (unlikely(!skb))
return NULL;
skb_mark_for_recycle(skb);
skb_reserve(skb, hr);
__skb_put(skb, size);
return skb;
}
/**
* idpf_rx_splitq_test_staterr - tests bits in Rx descriptor
* status and error fields
@@ -3169,13 +3134,15 @@ static bool idpf_rx_splitq_is_eop(struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_de
*/
static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget)
{
int total_rx_bytes = 0, total_rx_pkts = 0;
struct idpf_buf_queue *rx_bufq = NULL;
struct sk_buff *skb = rxq->skb;
struct libeth_rq_napi_stats rs = { };
u16 ntc = rxq->next_to_clean;
LIBETH_XDP_ONSTACK_BUFF(xdp);
libeth_xdp_init_buff(xdp, &rxq->xdp, &rxq->xdp_rxq);
/* Process Rx packets bounded by budget */
while (likely(total_rx_pkts < budget)) {
while (likely(rs.packets < budget)) {
struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc;
struct libeth_fqe *hdr, *rx_buf = NULL;
struct idpf_sw_queue *refillq = NULL;
@@ -3241,7 +3208,7 @@ static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget)
hdr = &rx_bufq->hdr_buf[buf_id];
if (unlikely(!hdr_len && !skb)) {
if (unlikely(!hdr_len && !xdp->data)) {
hdr_len = idpf_rx_hsplit_wa(hdr, rx_buf, pkt_len);
/* If failed, drop both buffers by setting len to 0 */
pkt_len -= hdr_len ? : pkt_len;
@@ -3251,75 +3218,35 @@ static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget)
u64_stats_update_end(&rxq->stats_sync);
}
if (libeth_rx_sync_for_cpu(hdr, hdr_len)) {
skb = idpf_rx_build_skb(hdr, hdr_len);
if (!skb)
break;
u64_stats_update_begin(&rxq->stats_sync);
u64_stats_inc(&rxq->q_stats.hsplit_pkts);
u64_stats_update_end(&rxq->stats_sync);
}
if (libeth_xdp_process_buff(xdp, hdr, hdr_len))
rs.hsplit++;
hdr->netmem = 0;
payload:
if (!libeth_rx_sync_for_cpu(rx_buf, pkt_len))
goto skip_data;
if (skb)
idpf_rx_add_frag(rx_buf, skb, pkt_len);
else
skb = idpf_rx_build_skb(rx_buf, pkt_len);
/* exit if we failed to retrieve a buffer */
if (!skb)
break;
skip_data:
libeth_xdp_process_buff(xdp, rx_buf, pkt_len);
rx_buf->netmem = 0;
idpf_post_buf_refill(refillq, buf_id);
IDPF_RX_BUMP_NTC(rxq, ntc);
/* skip if it is non EOP desc */
if (!idpf_rx_splitq_is_eop(rx_desc) || unlikely(!skb))
if (!idpf_rx_splitq_is_eop(rx_desc) || unlikely(!xdp->data))
continue;
/* pad skb if needed (to make valid ethernet frame) */
if (eth_skb_pad(skb)) {
skb = NULL;
continue;
}
/* probably a little skewed due to removing CRC */
total_rx_bytes += skb->len;
/* protocol */
if (unlikely(idpf_rx_process_skb_fields(rxq, skb, rx_desc))) {
dev_kfree_skb_any(skb);
skb = NULL;
continue;
}
/* send completed skb up the stack */
napi_gro_receive(rxq->napi, skb);
skb = NULL;
/* update budget accounting */
total_rx_pkts++;
idpf_xdp_run_pass(xdp, rxq->napi, &rs, rx_desc);
}
rxq->next_to_clean = ntc;
libeth_xdp_save_buff(&rxq->xdp, xdp);
rxq->skb = skb;
u64_stats_update_begin(&rxq->stats_sync);
u64_stats_add(&rxq->q_stats.packets, total_rx_pkts);
u64_stats_add(&rxq->q_stats.bytes, total_rx_bytes);
u64_stats_add(&rxq->q_stats.packets, rs.packets);
u64_stats_add(&rxq->q_stats.bytes, rs.bytes);
u64_stats_add(&rxq->q_stats.hsplit_pkts, rs.hsplit);
u64_stats_update_end(&rxq->stats_sync);
/* guarantee a trip back through this routine if there was a failure */
return total_rx_pkts;
return rs.packets;
}
/**
drivers/net/ethernet/intel/idpf/idpf_txrx.h
+7 -10
View File
@@ -484,7 +484,7 @@ struct idpf_tx_queue_stats {
* @next_to_use: Next descriptor to use
* @next_to_clean: Next descriptor to clean
* @next_to_alloc: RX buffer to allocate at
* @skb: Pointer to the skb
* @xdp: XDP buffer with the current frame
* @cached_phc_time: Cached PHC time for the Rx queue
* @stats_sync: See struct u64_stats_sync
* @q_stats: See union idpf_rx_queue_stats
@@ -536,11 +536,11 @@ struct idpf_rx_queue {
__cacheline_group_end_aligned(read_mostly);
__cacheline_group_begin_aligned(read_write);
u16 next_to_use;
u16 next_to_clean;
u16 next_to_alloc;
u32 next_to_use;
u32 next_to_clean;
u32 next_to_alloc;
struct sk_buff *skb;
struct libeth_xdp_buff_stash xdp;
u64 cached_phc_time;
struct u64_stats_sync stats_sync;
@@ -563,8 +563,8 @@ struct idpf_rx_queue {
libeth_cacheline_set_assert(struct idpf_rx_queue,
ALIGN(64, __alignof(struct xdp_rxq_info)) +
sizeof(struct xdp_rxq_info),
72 + offsetof(struct idpf_rx_queue, q_stats) -
offsetofend(struct idpf_rx_queue, skb),
96 + offsetof(struct idpf_rx_queue, q_stats) -
offsetofend(struct idpf_rx_queue, cached_phc_time),
32);
/**
@@ -1047,9 +1047,6 @@ int idpf_config_rss(struct idpf_vport *vport);
int idpf_init_rss(struct idpf_vport *vport);
void idpf_deinit_rss(struct idpf_vport *vport);
int idpf_rx_bufs_init_all(struct idpf_vport *vport);
void idpf_rx_add_frag(struct idpf_rx_buf *rx_buf, struct sk_buff *skb,
unsigned int size);
struct sk_buff *idpf_rx_build_skb(const struct libeth_fqe *buf, u32 size);
void idpf_tx_buf_hw_update(struct idpf_tx_queue *tx_q, u32 val,
bool xmit_more);
unsigned int idpf_size_to_txd_count(unsigned int size);