devlink: Introduce controller number

A devlink port may be for a controller consist of PCI device.
A devlink instance holds ports of two types of controllers.
(1) controller discovered on same system where eswitch resides
This is the case where PCI PF/VF of a controller and devlink eswitch
instance both are located on a single system.
(2) controller located on external host system.
This is the case where a controller is located in one system and its
devlink eswitch ports are located in a different system.

When a devlink eswitch instance serves the devlink ports of both
controllers together, PCI PF/VF numbers may overlap.
Due to this a unique phys_port_name cannot be constructed.

For example in below such system controller-0 and controller-1, each has
PCI PF pf0 whose eswitch ports can be present in controller-0.
These results in phys_port_name as "pf0" for both.
Similar problem exists for VFs and upcoming Sub functions.

An example view of two controller systems:

             ---------------------------------------------------------
             |                                                       |
             |           --------- ---------         ------- ------- |
-----------  |           | vf(s) | | sf(s) |         |vf(s)| |sf(s)| |
| server  |  | -------   ----/---- ---/----- ------- ---/--- ---/--- |
| pci rc  |=== | pf0 |______/________/       | pf1 |___/_______/     |
| connect |  | -------                       -------                 |
-----------  |     | controller_num=1 (no eswitch)                   |
             ------|--------------------------------------------------
             (internal wire)
                   |
             ---------------------------------------------------------
             | devlink eswitch ports and reps                        |
             | ----------------------------------------------------- |
             | |ctrl-0 | ctrl-0 | ctrl-0 | ctrl-0 | ctrl-0 |ctrl-0 | |
             | |pf0    | pf0vfN | pf0sfN | pf1    | pf1vfN |pf1sfN | |
             | ----------------------------------------------------- |
             | |ctrl-1 | ctrl-1 | ctrl-1 | ctrl-1 | ctrl-1 |ctrl-1 | |
             | |pf1    | pf1vfN | pf1sfN | pf1    | pf1vfN |pf0sfN | |
             | ----------------------------------------------------- |
             |                                                       |
             |                                                       |
             |           --------- ---------         ------- ------- |
             |           | vf(s) | | sf(s) |         |vf(s)| |sf(s)| |
             | -------   ----/---- ---/----- ------- ---/--- ---/--- |
             | | pf0 |______/________/       | pf1 |___/_______/     |
             | -------                       -------                 |
             |                                                       |
             |  local controller_num=0 (eswitch)                     |
             ---------------------------------------------------------

An example devlink port for external controller with controller
number = 1 for a VF 1 of PF 0:

$ devlink port show pci/0000:06:00.0/2
pci/0000:06:00.0/2: type eth netdev ens2f0pf0vf1 flavour pcivf controller 1 pfnum 0 vfnum 1 external true splittable false
  function:
    hw_addr 00:00:00:00:00:00

$ devlink port show pci/0000:06:00.0/2 -jp
{
    "port": {
        "pci/0000:06:00.0/2": {
            "type": "eth",
            "netdev": "ens2f0pf0vf1",
            "flavour": "pcivf",
            "controller": 1,
            "pfnum": 0,
            "vfnum": 1,
            "external": true,
            "splittable": false,
            "function": {
                "hw_addr": "00:00:00:00:00:00"
            }
        }
    }
}

Signed-off-by: Parav Pandit <parav@nvidia.com>
Reviewed-by: Jiri Pirko <jiri@nvidia.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
index 5b3599c..135ee26 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
@@ -1210,11 +1210,13 @@ is_devlink_port_supported(const struct mlx5_core_dev *dev,
 static int register_devlink_port(struct mlx5_core_dev *dev,
                                 struct mlx5e_rep_priv *rpriv)
 {
+       struct mlx5_esw_offload *offloads = &dev->priv.eswitch->offloads;
        struct devlink *devlink = priv_to_devlink(dev);
        struct mlx5_eswitch_rep *rep = rpriv->rep;
        struct devlink_port_attrs attrs = {};
        struct netdev_phys_item_id ppid = {};
        unsigned int dl_port_index = 0;
+       u32 controller_num = 0;
        bool external;
        u16 pfnum;
 
@@ -1222,6 +1224,8 @@ static int register_devlink_port(struct mlx5_core_dev *dev,
                return 0;
 
        external = mlx5_core_is_ecpf_esw_manager(dev);
+       if (external)
+               controller_num = offloads->host_number + 1;
        mlx5e_rep_get_port_parent_id(rpriv->netdev, &ppid);
        dl_port_index = mlx5_esw_vport_to_devlink_port_index(dev, rep->vport);
        pfnum = PCI_FUNC(dev->pdev->devfn);
@@ -1234,11 +1238,12 @@ static int register_devlink_port(struct mlx5_core_dev *dev,
        } else if (rep->vport == MLX5_VPORT_PF) {
                memcpy(rpriv->dl_port.attrs.switch_id.id, &ppid.id[0], ppid.id_len);
                rpriv->dl_port.attrs.switch_id.id_len = ppid.id_len;
-               devlink_port_attrs_pci_pf_set(&rpriv->dl_port, pfnum, external);
+               devlink_port_attrs_pci_pf_set(&rpriv->dl_port, controller_num,
+                                             pfnum, external);
        } else if (mlx5_eswitch_is_vf_vport(dev->priv.eswitch, rpriv->rep->vport)) {
                memcpy(rpriv->dl_port.attrs.switch_id.id, &ppid.id[0], ppid.id_len);
                rpriv->dl_port.attrs.switch_id.id_len = ppid.id_len;
-               devlink_port_attrs_pci_vf_set(&rpriv->dl_port,
+               devlink_port_attrs_pci_vf_set(&rpriv->dl_port, controller_num,
                                              pfnum, rep->vport - 1, external);
        }
        return devlink_port_register(devlink, &rpriv->dl_port, dl_port_index);

diff --git a/include/net/devlink.h b/include/net/devlink.h
index 2dad8c9..eaec0a8 100644 (file)
--- a/include/net/devlink.h
+++ b/include/net/devlink.h
@@ -59,21 +59,25 @@ struct devlink_port_phys_attrs {
 
 /**
  * struct devlink_port_pci_pf_attrs - devlink port's PCI PF attributes
+ * @controller: Associated controller number
  * @pf: Associated PCI PF number for this port.
  * @external: when set, indicates if a port is for an external controller
  */
 struct devlink_port_pci_pf_attrs {
+       u32 controller;
        u16 pf;
        u8 external:1;
 };
 
 /**
  * struct devlink_port_pci_vf_attrs - devlink port's PCI VF attributes
+ * @controller: Associated controller number
  * @pf: Associated PCI PF number for this port.
  * @vf: Associated PCI VF for of the PCI PF for this port.
  * @external: when set, indicates if a port is for an external controller
  */
 struct devlink_port_pci_vf_attrs {
+       u32 controller;
        u16 pf;
        u16 vf;
        u8 external:1;
@@ -1219,8 +1223,9 @@ void devlink_port_type_ib_set(struct devlink_port *devlink_port,
 void devlink_port_type_clear(struct devlink_port *devlink_port);
 void devlink_port_attrs_set(struct devlink_port *devlink_port,
                            struct devlink_port_attrs *devlink_port_attrs);
-void devlink_port_attrs_pci_pf_set(struct devlink_port *devlink_port, u16 pf, bool external);
-void devlink_port_attrs_pci_vf_set(struct devlink_port *devlink_port,
+void devlink_port_attrs_pci_pf_set(struct devlink_port *devlink_port, u32 controller,
+                                  u16 pf, bool external);
+void devlink_port_attrs_pci_vf_set(struct devlink_port *devlink_port, u32 controller,
                                   u16 pf, u16 vf, bool external);
 int devlink_sb_register(struct devlink *devlink, unsigned int sb_index,
                        u32 size, u16 ingress_pools_count,

diff --git a/include/uapi/linux/devlink.h b/include/uapi/linux/devlink.h
index 40823ed..40d3514 100644 (file)
--- a/include/uapi/linux/devlink.h
+++ b/include/uapi/linux/devlink.h
@@ -459,6 +459,7 @@ enum devlink_attr {
        DEVLINK_ATTR_PORT_SPLITTABLE,                   /* u8 */
 
        DEVLINK_ATTR_PORT_EXTERNAL,             /* u8 */
+       DEVLINK_ATTR_PORT_CONTROLLER_NUMBER,    /* u32 */
        /* add new attributes above here, update the policy in devlink.c */
 
        __DEVLINK_ATTR_MAX,

diff --git a/net/core/devlink.c b/net/core/devlink.c
index 6f5f853..9cf5b11 100644 (file)
--- a/net/core/devlink.c
+++ b/net/core/devlink.c
@@ -523,17 +523,18 @@ static int devlink_nl_port_attrs_put(struct sk_buff *msg,
                return -EMSGSIZE;
        switch (devlink_port->attrs.flavour) {
        case DEVLINK_PORT_FLAVOUR_PCI_PF:
-               if (nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_PF_NUMBER,
-                               attrs->pci_pf.pf))
+               if (nla_put_u32(msg, DEVLINK_ATTR_PORT_CONTROLLER_NUMBER,
+                               attrs->pci_pf.controller) ||
+                   nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_PF_NUMBER, attrs->pci_pf.pf))
                        return -EMSGSIZE;
                if (nla_put_u8(msg, DEVLINK_ATTR_PORT_EXTERNAL, attrs->pci_pf.external))
                        return -EMSGSIZE;
                break;
        case DEVLINK_PORT_FLAVOUR_PCI_VF:
-               if (nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_PF_NUMBER,
-                               attrs->pci_vf.pf) ||
-                   nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_VF_NUMBER,
-                               attrs->pci_vf.vf))
+               if (nla_put_u32(msg, DEVLINK_ATTR_PORT_CONTROLLER_NUMBER,
+                               attrs->pci_vf.controller) ||
+                   nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_PF_NUMBER, attrs->pci_vf.pf) ||
+                   nla_put_u16(msg, DEVLINK_ATTR_PORT_PCI_VF_NUMBER, attrs->pci_vf.vf))
                        return -EMSGSIZE;
                if (nla_put_u8(msg, DEVLINK_ATTR_PORT_EXTERNAL, attrs->pci_vf.external))
                        return -EMSGSIZE;
@@ -7719,10 +7720,12 @@ EXPORT_SYMBOL_GPL(devlink_port_attrs_set);
  *     devlink_port_attrs_pci_pf_set - Set PCI PF port attributes
  *
  *     @devlink_port: devlink port
+ *     @controller: associated controller number for the devlink port instance
  *     @pf: associated PF for the devlink port instance
  *     @external: indicates if the port is for an external controller
  */
-void devlink_port_attrs_pci_pf_set(struct devlink_port *devlink_port, u16 pf, bool external)
+void devlink_port_attrs_pci_pf_set(struct devlink_port *devlink_port, u32 controller,
+                                  u16 pf, bool external)
 {
        struct devlink_port_attrs *attrs = &devlink_port->attrs;
        int ret;
@@ -7731,7 +7734,7 @@ void devlink_port_attrs_pci_pf_set(struct devlink_port *devlink_port, u16 pf, bo
                                       DEVLINK_PORT_FLAVOUR_PCI_PF);
        if (ret)
                return;
-
+       attrs->pci_pf.controller = controller;
        attrs->pci_pf.pf = pf;
        attrs->pci_pf.external = external;
 }
@@ -7741,11 +7744,12 @@ EXPORT_SYMBOL_GPL(devlink_port_attrs_pci_pf_set);
  *     devlink_port_attrs_pci_vf_set - Set PCI VF port attributes
  *
  *     @devlink_port: devlink port
+ *     @controller: associated controller number for the devlink port instance
  *     @pf: associated PF for the devlink port instance
  *     @vf: associated VF of a PF for the devlink port instance
  *     @external: indicates if the port is for an external controller
  */
-void devlink_port_attrs_pci_vf_set(struct devlink_port *devlink_port,
+void devlink_port_attrs_pci_vf_set(struct devlink_port *devlink_port, u32 controller,
                                   u16 pf, u16 vf, bool external)
 {
        struct devlink_port_attrs *attrs = &devlink_port->attrs;
@@ -7755,6 +7759,7 @@ void devlink_port_attrs_pci_vf_set(struct devlink_port *devlink_port,
                                       DEVLINK_PORT_FLAVOUR_PCI_VF);
        if (ret)
                return;
+       attrs->pci_vf.controller = controller;
        attrs->pci_vf.pf = pf;
        attrs->pci_vf.vf = vf;
        attrs->pci_vf.external = external;