struct veth_rq {
struct napi_struct xdp_napi;
+ struct napi_struct __rcu *napi; /* points to xdp_napi when the latter is initialized */
struct net_device *dev;
struct bpf_prog __rcu *xdp_prog;
struct xdp_mem_info xdp_mem;
}
}
+static void veth_get_channels(struct net_device *dev,
+ struct ethtool_channels *channels)
+{
+ channels->tx_count = dev->real_num_tx_queues;
+ channels->rx_count = dev->real_num_rx_queues;
+ channels->max_tx = dev->num_tx_queues;
+ channels->max_rx = dev->num_rx_queues;
+}
+
+static int veth_set_channels(struct net_device *dev,
+ struct ethtool_channels *ch);
+
static const struct ethtool_ops veth_ethtool_ops = {
.get_drvinfo = veth_get_drvinfo,
.get_link = ethtool_op_get_link,
.get_ethtool_stats = veth_get_ethtool_stats,
.get_link_ksettings = veth_get_link_ksettings,
.get_ts_info = ethtool_op_get_ts_info,
+ .get_channels = veth_get_channels,
+ .set_channels = veth_set_channels,
};
/* general routines */
netif_rx(skb);
}
+/* return true if the specified skb has chances of GRO aggregation
+ * Don't strive for accuracy, but try to avoid GRO overhead in the most
+ * common scenarios.
+ * When XDP is enabled, all traffic is considered eligible, as the xmit
+ * device has TSO off.
+ * When TSO is enabled on the xmit device, we are likely interested only
+ * in UDP aggregation, explicitly check for that if the skb is suspected
+ * - the sock_wfree destructor is used by UDP, ICMP and XDP sockets -
+ * to belong to locally generated UDP traffic.
+ */
+static bool veth_skb_is_eligible_for_gro(const struct net_device *dev,
+ const struct net_device *rcv,
+ const struct sk_buff *skb)
+{
+ return !(dev->features & NETIF_F_ALL_TSO) ||
+ (skb->destructor == sock_wfree &&
+ rcv->features & (NETIF_F_GRO_FRAGLIST | NETIF_F_GRO_UDP_FWD));
+}
+
static netdev_tx_t veth_xmit(struct sk_buff *skb, struct net_device *dev)
{
struct veth_priv *rcv_priv, *priv = netdev_priv(dev);
struct veth_rq *rq = NULL;
struct net_device *rcv;
int length = skb->len;
- bool rcv_xdp = false;
+ bool use_napi = false;
int rxq;
rcu_read_lock();
rxq = skb_get_queue_mapping(skb);
if (rxq < rcv->real_num_rx_queues) {
rq = &rcv_priv->rq[rxq];
- rcv_xdp = rcu_access_pointer(rq->xdp_prog);
+
+ /* The napi pointer is available when an XDP program is
+ * attached or when GRO is enabled
+ * Don't bother with napi/GRO if the skb can't be aggregated
+ */
+ use_napi = rcu_access_pointer(rq->napi) &&
+ veth_skb_is_eligible_for_gro(dev, rcv, skb);
skb_record_rx_queue(skb, rxq);
}
skb_tx_timestamp(skb);
- if (likely(veth_forward_skb(rcv, skb, rq, rcv_xdp) == NET_RX_SUCCESS)) {
- if (!rcv_xdp)
+ if (likely(veth_forward_skb(rcv, skb, rq, use_napi) == NET_RX_SUCCESS)) {
+ if (!use_napi)
dev_lstats_add(dev, length);
} else {
drop:
atomic64_inc(&priv->dropped);
}
- if (rcv_xdp)
+ if (use_napi)
__veth_xdp_flush(rq);
rcu_read_unlock();
u32 flags, bool ndo_xmit)
{
struct veth_priv *rcv_priv, *priv = netdev_priv(dev);
- int i, ret = -ENXIO, drops = 0;
+ int i, ret = -ENXIO, nxmit = 0;
struct net_device *rcv;
unsigned int max_len;
struct veth_rq *rq;
rcv_priv = netdev_priv(rcv);
rq = &rcv_priv->rq[veth_select_rxq(rcv)];
- /* Non-NULL xdp_prog ensures that xdp_ring is initialized on receive
- * side. This means an XDP program is loaded on the peer and the peer
- * device is up.
+ /* The napi pointer is set if NAPI is enabled, which ensures that
+ * xdp_ring is initialized on receive side and the peer device is up.
*/
- if (!rcu_access_pointer(rq->xdp_prog))
+ if (!rcu_access_pointer(rq->napi))
goto out;
max_len = rcv->mtu + rcv->hard_header_len + VLAN_HLEN;
void *ptr = veth_xdp_to_ptr(frame);
if (unlikely(frame->len > max_len ||
- __ptr_ring_produce(&rq->xdp_ring, ptr))) {
- xdp_return_frame_rx_napi(frame);
- drops++;
- }
+ __ptr_ring_produce(&rq->xdp_ring, ptr)))
+ break;
+ nxmit++;
}
spin_unlock(&rq->xdp_ring.producer_lock);
if (flags & XDP_XMIT_FLUSH)
__veth_xdp_flush(rq);
- ret = n - drops;
+ ret = nxmit;
if (ndo_xmit) {
u64_stats_update_begin(&rq->stats.syncp);
- rq->stats.vs.peer_tq_xdp_xmit += n - drops;
- rq->stats.vs.peer_tq_xdp_xmit_err += drops;
+ rq->stats.vs.peer_tq_xdp_xmit += nxmit;
+ rq->stats.vs.peer_tq_xdp_xmit_err += n - nxmit;
u64_stats_update_end(&rq->stats.syncp);
}
static void veth_xdp_flush_bq(struct veth_rq *rq, struct veth_xdp_tx_bq *bq)
{
- int sent, i, err = 0;
+ int sent, i, err = 0, drops;
sent = veth_xdp_xmit(rq->dev, bq->count, bq->q, 0, false);
if (sent < 0) {
err = sent;
sent = 0;
- for (i = 0; i < bq->count; i++)
- xdp_return_frame(bq->q[i]);
}
- trace_xdp_bulk_tx(rq->dev, sent, bq->count - sent, err);
+
+ for (i = sent; unlikely(i < bq->count); i++)
+ xdp_return_frame(bq->q[i]);
+
+ drops = bq->count - sent;
+ trace_xdp_bulk_tx(rq->dev, sent, drops, err);
u64_stats_update_begin(&rq->stats.syncp);
rq->stats.vs.xdp_tx += sent;
- rq->stats.vs.xdp_tx_err += bq->count - sent;
+ rq->stats.vs.xdp_tx_err += drops;
u64_stats_update_end(&rq->stats.syncp);
bq->count = 0;
int mac_len, delta, off;
struct xdp_buff xdp;
- skb_orphan(skb);
+ skb_prepare_for_gro(skb);
rcu_read_lock();
xdp_prog = rcu_dereference(rq->xdp_prog);
return done;
}
-static int veth_napi_add(struct net_device *dev)
+static int __veth_napi_enable_range(struct net_device *dev, int start, int end)
{
struct veth_priv *priv = netdev_priv(dev);
int err, i;
- for (i = 0; i < dev->real_num_rx_queues; i++) {
+ for (i = start; i < end; i++) {
struct veth_rq *rq = &priv->rq[i];
err = ptr_ring_init(&rq->xdp_ring, VETH_RING_SIZE, GFP_KERNEL);
goto err_xdp_ring;
}
- for (i = 0; i < dev->real_num_rx_queues; i++) {
+ for (i = start; i < end; i++) {
struct veth_rq *rq = &priv->rq[i];
napi_enable(&rq->xdp_napi);
+ rcu_assign_pointer(priv->rq[i].napi, &priv->rq[i].xdp_napi);
}
return 0;
+
err_xdp_ring:
- for (i--; i >= 0; i--)
+ for (i--; i >= start; i--)
ptr_ring_cleanup(&priv->rq[i].xdp_ring, veth_ptr_free);
return err;
}
-static void veth_napi_del(struct net_device *dev)
+static int __veth_napi_enable(struct net_device *dev)
+{
+ return __veth_napi_enable_range(dev, 0, dev->real_num_rx_queues);
+}
+
+static void veth_napi_del_range(struct net_device *dev, int start, int end)
{
struct veth_priv *priv = netdev_priv(dev);
int i;
- for (i = 0; i < dev->real_num_rx_queues; i++) {
+ for (i = start; i < end; i++) {
struct veth_rq *rq = &priv->rq[i];
+ rcu_assign_pointer(priv->rq[i].napi, NULL);
napi_disable(&rq->xdp_napi);
__netif_napi_del(&rq->xdp_napi);
}
synchronize_net();
- for (i = 0; i < dev->real_num_rx_queues; i++) {
+ for (i = start; i < end; i++) {
struct veth_rq *rq = &priv->rq[i];
rq->rx_notify_masked = false;
}
}
-static int veth_enable_xdp(struct net_device *dev)
+static void veth_napi_del(struct net_device *dev)
+{
+ veth_napi_del_range(dev, 0, dev->real_num_rx_queues);
+}
+
+static bool veth_gro_requested(const struct net_device *dev)
+{
+ return !!(dev->wanted_features & NETIF_F_GRO);
+}
+
+static int veth_enable_xdp_range(struct net_device *dev, int start, int end,
+ bool napi_already_on)
{
struct veth_priv *priv = netdev_priv(dev);
int err, i;
- if (!xdp_rxq_info_is_reg(&priv->rq[0].xdp_rxq)) {
- for (i = 0; i < dev->real_num_rx_queues; i++) {
- struct veth_rq *rq = &priv->rq[i];
+ for (i = start; i < end; i++) {
+ struct veth_rq *rq = &priv->rq[i];
+ if (!napi_already_on)
netif_napi_add(dev, &rq->xdp_napi, veth_poll, NAPI_POLL_WEIGHT);
- err = xdp_rxq_info_reg(&rq->xdp_rxq, dev, i, rq->xdp_napi.napi_id);
- if (err < 0)
- goto err_rxq_reg;
-
- err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,
- MEM_TYPE_PAGE_SHARED,
- NULL);
- if (err < 0)
- goto err_reg_mem;
-
- /* Save original mem info as it can be overwritten */
- rq->xdp_mem = rq->xdp_rxq.mem;
- }
-
- err = veth_napi_add(dev);
- if (err)
+ err = xdp_rxq_info_reg(&rq->xdp_rxq, dev, i, rq->xdp_napi.napi_id);
+ if (err < 0)
goto err_rxq_reg;
- }
- for (i = 0; i < dev->real_num_rx_queues; i++)
- rcu_assign_pointer(priv->rq[i].xdp_prog, priv->_xdp_prog);
+ err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,
+ MEM_TYPE_PAGE_SHARED,
+ NULL);
+ if (err < 0)
+ goto err_reg_mem;
+ /* Save original mem info as it can be overwritten */
+ rq->xdp_mem = rq->xdp_rxq.mem;
+ }
return 0;
+
err_reg_mem:
xdp_rxq_info_unreg(&priv->rq[i].xdp_rxq);
err_rxq_reg:
- for (i--; i >= 0; i--) {
+ for (i--; i >= start; i--) {
struct veth_rq *rq = &priv->rq[i];
xdp_rxq_info_unreg(&rq->xdp_rxq);
- netif_napi_del(&rq->xdp_napi);
+ if (!napi_already_on)
+ netif_napi_del(&rq->xdp_napi);
}
return err;
}
+static void veth_disable_xdp_range(struct net_device *dev, int start, int end,
+ bool delete_napi)
+{
+ struct veth_priv *priv = netdev_priv(dev);
+ int i;
+
+ for (i = start; i < end; i++) {
+ struct veth_rq *rq = &priv->rq[i];
+
+ rq->xdp_rxq.mem = rq->xdp_mem;
+ xdp_rxq_info_unreg(&rq->xdp_rxq);
+
+ if (delete_napi)
+ netif_napi_del(&rq->xdp_napi);
+ }
+}
+
+static int veth_enable_xdp(struct net_device *dev)
+{
+ bool napi_already_on = veth_gro_requested(dev) && (dev->flags & IFF_UP);
+ struct veth_priv *priv = netdev_priv(dev);
+ int err, i;
+
+ if (!xdp_rxq_info_is_reg(&priv->rq[0].xdp_rxq)) {
+ err = veth_enable_xdp_range(dev, 0, dev->real_num_rx_queues, napi_already_on);
+ if (err)
+ return err;
+
+ if (!napi_already_on) {
+ err = __veth_napi_enable(dev);
+ if (err) {
+ veth_disable_xdp_range(dev, 0, dev->real_num_rx_queues, true);
+ return err;
+ }
+
+ if (!veth_gro_requested(dev)) {
+ /* user-space did not require GRO, but adding XDP
+ * is supposed to get GRO working
+ */
+ dev->features |= NETIF_F_GRO;
+ netdev_features_change(dev);
+ }
+ }
+ }
+
+ for (i = 0; i < dev->real_num_rx_queues; i++) {
+ rcu_assign_pointer(priv->rq[i].xdp_prog, priv->_xdp_prog);
+ rcu_assign_pointer(priv->rq[i].napi, &priv->rq[i].xdp_napi);
+ }
+
+ return 0;
+}
+
static void veth_disable_xdp(struct net_device *dev)
{
struct veth_priv *priv = netdev_priv(dev);
for (i = 0; i < dev->real_num_rx_queues; i++)
rcu_assign_pointer(priv->rq[i].xdp_prog, NULL);
- veth_napi_del(dev);
- for (i = 0; i < dev->real_num_rx_queues; i++) {
+
+ if (!netif_running(dev) || !veth_gro_requested(dev)) {
+ veth_napi_del(dev);
+
+ /* if user-space did not require GRO, since adding XDP
+ * enabled it, clear it now
+ */
+ if (!veth_gro_requested(dev) && netif_running(dev)) {
+ dev->features &= ~NETIF_F_GRO;
+ netdev_features_change(dev);
+ }
+ }
+
+ veth_disable_xdp_range(dev, 0, dev->real_num_rx_queues, false);
+}
+
+static int veth_napi_enable_range(struct net_device *dev, int start, int end)
+{
+ struct veth_priv *priv = netdev_priv(dev);
+ int err, i;
+
+ for (i = start; i < end; i++) {
struct veth_rq *rq = &priv->rq[i];
- rq->xdp_rxq.mem = rq->xdp_mem;
- xdp_rxq_info_unreg(&rq->xdp_rxq);
+ netif_napi_add(dev, &rq->xdp_napi, veth_poll, NAPI_POLL_WEIGHT);
+ }
+
+ err = __veth_napi_enable_range(dev, start, end);
+ if (err) {
+ for (i = start; i < end; i++) {
+ struct veth_rq *rq = &priv->rq[i];
+
+ netif_napi_del(&rq->xdp_napi);
+ }
+ return err;
+ }
+ return err;
+}
+
+static int veth_napi_enable(struct net_device *dev)
+{
+ return veth_napi_enable_range(dev, 0, dev->real_num_rx_queues);
+}
+
+static void veth_disable_range_safe(struct net_device *dev, int start, int end)
+{
+ struct veth_priv *priv = netdev_priv(dev);
+
+ if (start >= end)
+ return;
+
+ if (priv->_xdp_prog) {
+ veth_napi_del_range(dev, start, end);
+ veth_disable_xdp_range(dev, start, end, false);
+ } else if (veth_gro_requested(dev)) {
+ veth_napi_del_range(dev, start, end);
+ }
+}
+
+static int veth_enable_range_safe(struct net_device *dev, int start, int end)
+{
+ struct veth_priv *priv = netdev_priv(dev);
+ int err;
+
+ if (start >= end)
+ return 0;
+
+ if (priv->_xdp_prog) {
+ /* these channels are freshly initialized, napi is not on there even
+ * when GRO is requeste
+ */
+ err = veth_enable_xdp_range(dev, start, end, false);
+ if (err)
+ return err;
+
+ err = __veth_napi_enable_range(dev, start, end);
+ if (err) {
+ /* on error always delete the newly added napis */
+ veth_disable_xdp_range(dev, start, end, true);
+ return err;
+ }
+ } else if (veth_gro_requested(dev)) {
+ return veth_napi_enable_range(dev, start, end);
}
+ return 0;
+}
+
+static int veth_set_channels(struct net_device *dev,
+ struct ethtool_channels *ch)
+{
+ struct veth_priv *priv = netdev_priv(dev);
+ unsigned int old_rx_count, new_rx_count;
+ struct veth_priv *peer_priv;
+ struct net_device *peer;
+ int err;
+
+ /* sanity check. Upper bounds are already enforced by the caller */
+ if (!ch->rx_count || !ch->tx_count)
+ return -EINVAL;
+
+ /* avoid braking XDP, if that is enabled */
+ peer = rtnl_dereference(priv->peer);
+ peer_priv = peer ? netdev_priv(peer) : NULL;
+ if (priv->_xdp_prog && peer && ch->rx_count < peer->real_num_tx_queues)
+ return -EINVAL;
+
+ if (peer && peer_priv && peer_priv->_xdp_prog && ch->tx_count > peer->real_num_rx_queues)
+ return -EINVAL;
+
+ old_rx_count = dev->real_num_rx_queues;
+ new_rx_count = ch->rx_count;
+ if (netif_running(dev)) {
+ /* turn device off */
+ netif_carrier_off(dev);
+ if (peer)
+ netif_carrier_off(peer);
+
+ /* try to allocate new resurces, as needed*/
+ err = veth_enable_range_safe(dev, old_rx_count, new_rx_count);
+ if (err)
+ goto out;
+ }
+
+ err = netif_set_real_num_rx_queues(dev, ch->rx_count);
+ if (err)
+ goto revert;
+
+ err = netif_set_real_num_tx_queues(dev, ch->tx_count);
+ if (err) {
+ int err2 = netif_set_real_num_rx_queues(dev, old_rx_count);
+
+ /* this error condition could happen only if rx and tx change
+ * in opposite directions (e.g. tx nr raises, rx nr decreases)
+ * and we can't do anything to fully restore the original
+ * status
+ */
+ if (err2)
+ pr_warn("Can't restore rx queues config %d -> %d %d",
+ new_rx_count, old_rx_count, err2);
+ else
+ goto revert;
+ }
+
+out:
+ if (netif_running(dev)) {
+ /* note that we need to swap the arguments WRT the enable part
+ * to identify the range we have to disable
+ */
+ veth_disable_range_safe(dev, new_rx_count, old_rx_count);
+ netif_carrier_on(dev);
+ if (peer)
+ netif_carrier_on(peer);
+ }
+ return err;
+
+revert:
+ new_rx_count = old_rx_count;
+ old_rx_count = ch->rx_count;
+ goto out;
}
static int veth_open(struct net_device *dev)
err = veth_enable_xdp(dev);
if (err)
return err;
+ } else if (veth_gro_requested(dev)) {
+ err = veth_napi_enable(dev);
+ if (err)
+ return err;
}
if (peer->flags & IFF_UP) {
if (priv->_xdp_prog)
veth_disable_xdp(dev);
+ else if (veth_gro_requested(dev))
+ veth_napi_del(dev);
return 0;
}
if (peer_priv->_xdp_prog)
features &= ~NETIF_F_GSO_SOFTWARE;
}
+ if (priv->_xdp_prog)
+ features |= NETIF_F_GRO;
return features;
}
+static int veth_set_features(struct net_device *dev,
+ netdev_features_t features)
+{
+ netdev_features_t changed = features ^ dev->features;
+ struct veth_priv *priv = netdev_priv(dev);
+ int err;
+
+ if (!(changed & NETIF_F_GRO) || !(dev->flags & IFF_UP) || priv->_xdp_prog)
+ return 0;
+
+ if (features & NETIF_F_GRO) {
+ err = veth_napi_enable(dev);
+ if (err)
+ return err;
+ } else {
+ veth_napi_del(dev);
+ }
+ return 0;
+}
+
static void veth_set_rx_headroom(struct net_device *dev, int new_hr)
{
struct veth_priv *peer_priv, *priv = netdev_priv(dev);
#endif
.ndo_get_iflink = veth_get_iflink,
.ndo_fix_features = veth_fix_features,
+ .ndo_set_features = veth_set_features,
.ndo_features_check = passthru_features_check,
.ndo_set_rx_headroom = veth_set_rx_headroom,
.ndo_bpf = veth_xdp,
static struct rtnl_link_ops veth_link_ops;
+static void veth_disable_gro(struct net_device *dev)
+{
+ dev->features &= ~NETIF_F_GRO;
+ dev->wanted_features &= ~NETIF_F_GRO;
+ netdev_update_features(dev);
+}
+
+static int veth_init_queues(struct net_device *dev, struct nlattr *tb[])
+{
+ int err;
+
+ if (!tb[IFLA_NUM_TX_QUEUES] && dev->num_tx_queues > 1) {
+ err = netif_set_real_num_tx_queues(dev, 1);
+ if (err)
+ return err;
+ }
+ if (!tb[IFLA_NUM_RX_QUEUES] && dev->num_rx_queues > 1) {
+ err = netif_set_real_num_rx_queues(dev, 1);
+ if (err)
+ return err;
+ }
+ return 0;
+}
+
static int veth_newlink(struct net *src_net, struct net_device *dev,
struct nlattr *tb[], struct nlattr *data[],
struct netlink_ext_ack *extack)
if (err < 0)
goto err_register_peer;
+ /* keep GRO disabled by default to be consistent with the established
+ * veth behavior
+ */
+ veth_disable_gro(peer);
netif_carrier_off(peer);
err = rtnl_configure_link(peer, ifmp);
priv = netdev_priv(dev);
rcu_assign_pointer(priv->peer, peer);
+ err = veth_init_queues(dev, tb);
+ if (err)
+ goto err_queues;
priv = netdev_priv(peer);
rcu_assign_pointer(priv->peer, dev);
+ err = veth_init_queues(peer, tb);
+ if (err)
+ goto err_queues;
+ veth_disable_gro(dev);
return 0;
+err_queues:
+ unregister_netdevice(dev);
err_register_dev:
/* nothing to do */
err_configure_peer:
return peer ? dev_net(peer) : dev_net(dev);
}
+static unsigned int veth_get_num_queues(void)
+{
+ /* enforce the same queue limit as rtnl_create_link */
+ int queues = num_possible_cpus();
+
+ if (queues > 4096)
+ queues = 4096;
+ return queues;
+}
+
static struct rtnl_link_ops veth_link_ops = {
.kind = DRV_NAME,
.priv_size = sizeof(struct veth_priv),
.policy = veth_policy,
.maxtype = VETH_INFO_MAX,
.get_link_net = veth_get_link_net,
+ .get_num_tx_queues = veth_get_num_queues,
+ .get_num_rx_queues = veth_get_num_queues,
};
/*
projects / linux-2.6-microblaze.git / blobdiff