drivers/net/ethernet/marvell/octeontx2/af/cgx.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Marvell OcteonTx2 CGX driver
   3  *
   4  * Copyright (C) 2018 Marvell.
   5  *
   6  */
   7
   8 #include <linux/acpi.h>
   9 #include <linux/module.h>
  10 #include <linux/interrupt.h>
  11 #include <linux/pci.h>
  12 #include <linux/netdevice.h>
  13 #include <linux/etherdevice.h>
  14 #include <linux/ethtool.h>
  15 #include <linux/phy.h>
  16 #include <linux/of.h>
  17 #include <linux/of_mdio.h>
  18 #include <linux/of_net.h>
  19
  20 #include "cgx.h"
  21 #include "rvu.h"
  22 #include "lmac_common.h"
  23
  24 #define DRV_NAME        "Marvell-CGX/RPM"
  25 #define DRV_STRING      "Marvell CGX/RPM Driver"
  26
  27 static LIST_HEAD(cgx_list);
  28
  29 /* Convert firmware speed encoding to user format(Mbps) */
  30 static const u32 cgx_speed_mbps[CGX_LINK_SPEED_MAX] = {
  31         [CGX_LINK_NONE] = 0,
  32         [CGX_LINK_10M] = 10,
  33         [CGX_LINK_100M] = 100,
  34         [CGX_LINK_1G] = 1000,
  35         [CGX_LINK_2HG] = 2500,
  36         [CGX_LINK_5G] = 5000,
  37         [CGX_LINK_10G] = 10000,
  38         [CGX_LINK_20G] = 20000,
  39         [CGX_LINK_25G] = 25000,
  40         [CGX_LINK_40G] = 40000,
  41         [CGX_LINK_50G] = 50000,
  42         [CGX_LINK_80G] = 80000,
  43         [CGX_LINK_100G] = 100000,
  44 };
  45
  46 /* Convert firmware lmac type encoding to string */
  47 static const char *cgx_lmactype_string[LMAC_MODE_MAX] = {
  48         [LMAC_MODE_SGMII] = "SGMII",
  49         [LMAC_MODE_XAUI] = "XAUI",
  50         [LMAC_MODE_RXAUI] = "RXAUI",
  51         [LMAC_MODE_10G_R] = "10G_R",
  52         [LMAC_MODE_40G_R] = "40G_R",
  53         [LMAC_MODE_QSGMII] = "QSGMII",
  54         [LMAC_MODE_25G_R] = "25G_R",
  55         [LMAC_MODE_50G_R] = "50G_R",
  56         [LMAC_MODE_100G_R] = "100G_R",
  57         [LMAC_MODE_USXGMII] = "USXGMII",
  58 };
  59
  60 /* CGX PHY management internal APIs */
  61 static int cgx_fwi_link_change(struct cgx *cgx, int lmac_id, bool en);
  62
  63 /* Supported devices */
  64 static const struct pci_device_id cgx_id_table[] = {
  65         { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_OCTEONTX2_CGX) },
  66         { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CN10K_RPM) },
  67         { 0, }  /* end of table */
  68 };
  69
  70 MODULE_DEVICE_TABLE(pci, cgx_id_table);
  71
  72 static bool is_dev_rpm(void *cgxd)
  73 {
  74         struct cgx *cgx = cgxd;
  75
  76         return (cgx->pdev->device == PCI_DEVID_CN10K_RPM);
  77 }
  78
  79 bool is_lmac_valid(struct cgx *cgx, int lmac_id)
  80 {
  81         if (!cgx || lmac_id < 0 || lmac_id >= MAX_LMAC_PER_CGX)
  82                 return false;
  83         return test_bit(lmac_id, &cgx->lmac_bmap);
  84 }
  85
  86 /* Helper function to get sequential index
  87  * given the enabled LMAC of a CGX
  88  */
  89 static int get_sequence_id_of_lmac(struct cgx *cgx, int lmac_id)
  90 {
  91         int tmp, id = 0;
  92
  93         for_each_set_bit(tmp, &cgx->lmac_bmap, MAX_LMAC_PER_CGX) {
  94                 if (tmp == lmac_id)
  95                         break;
  96                 id++;
  97         }
  98
  99         return id;
 100 }
 101
 102 struct mac_ops *get_mac_ops(void *cgxd)
 103 {
 104         if (!cgxd)
 105                 return cgxd;
 106
 107         return ((struct cgx *)cgxd)->mac_ops;
 108 }
 109
 110 void cgx_write(struct cgx *cgx, u64 lmac, u64 offset, u64 val)
 111 {
 112         writeq(val, cgx->reg_base + (lmac << cgx->mac_ops->lmac_offset) +
 113                offset);
 114 }
 115
 116 u64 cgx_read(struct cgx *cgx, u64 lmac, u64 offset)
 117 {
 118         return readq(cgx->reg_base + (lmac << cgx->mac_ops->lmac_offset) +
 119                      offset);
 120 }
 121
 122 struct lmac *lmac_pdata(u8 lmac_id, struct cgx *cgx)
 123 {
 124         if (!cgx || lmac_id >= MAX_LMAC_PER_CGX)
 125                 return NULL;
 126
 127         return cgx->lmac_idmap[lmac_id];
 128 }
 129
 130 int cgx_get_cgxcnt_max(void)
 131 {
 132         struct cgx *cgx_dev;
 133         int idmax = -ENODEV;
 134
 135         list_for_each_entry(cgx_dev, &cgx_list, cgx_list)
 136                 if (cgx_dev->cgx_id > idmax)
 137                         idmax = cgx_dev->cgx_id;
 138
 139         if (idmax < 0)
 140                 return 0;
 141
 142         return idmax + 1;
 143 }
 144
 145 int cgx_get_lmac_cnt(void *cgxd)
 146 {
 147         struct cgx *cgx = cgxd;
 148
 149         if (!cgx)
 150                 return -ENODEV;
 151
 152         return cgx->lmac_count;
 153 }
 154
 155 void *cgx_get_pdata(int cgx_id)
 156 {
 157         struct cgx *cgx_dev;
 158
 159         list_for_each_entry(cgx_dev, &cgx_list, cgx_list) {
 160                 if (cgx_dev->cgx_id == cgx_id)
 161                         return cgx_dev;
 162         }
 163         return NULL;
 164 }
 165
 166 void cgx_lmac_write(int cgx_id, int lmac_id, u64 offset, u64 val)
 167 {
 168         struct cgx *cgx_dev = cgx_get_pdata(cgx_id);
 169
 170         cgx_write(cgx_dev, lmac_id, offset, val);
 171 }
 172
 173 u64 cgx_lmac_read(int cgx_id, int lmac_id, u64 offset)
 174 {
 175         struct cgx *cgx_dev = cgx_get_pdata(cgx_id);
 176
 177         return cgx_read(cgx_dev, lmac_id, offset);
 178 }
 179
 180 int cgx_get_cgxid(void *cgxd)
 181 {
 182         struct cgx *cgx = cgxd;
 183
 184         if (!cgx)
 185                 return -EINVAL;
 186
 187         return cgx->cgx_id;
 188 }
 189
 190 u8 cgx_lmac_get_p2x(int cgx_id, int lmac_id)
 191 {
 192         struct cgx *cgx_dev = cgx_get_pdata(cgx_id);
 193         u64 cfg;
 194
 195         cfg = cgx_read(cgx_dev, lmac_id, CGXX_CMRX_CFG);
 196
 197         return (cfg & CMR_P2X_SEL_MASK) >> CMR_P2X_SEL_SHIFT;
 198 }
 199
 200 /* Ensure the required lock for event queue(where asynchronous events are
 201  * posted) is acquired before calling this API. Else an asynchronous event(with
 202  * latest link status) can reach the destination before this function returns
 203  * and could make the link status appear wrong.
 204  */
 205 int cgx_get_link_info(void *cgxd, int lmac_id,
 206                       struct cgx_link_user_info *linfo)
 207 {
 208         struct lmac *lmac = lmac_pdata(lmac_id, cgxd);
 209
 210         if (!lmac)
 211                 return -ENODEV;
 212
 213         *linfo = lmac->link_info;
 214         return 0;
 215 }
 216
 217 static u64 mac2u64 (u8 *mac_addr)
 218 {
 219         u64 mac = 0;
 220         int index;
 221
 222         for (index = ETH_ALEN - 1; index >= 0; index--)
 223                 mac |= ((u64)*mac_addr++) << (8 * index);
 224         return mac;
 225 }
 226
 227 static void cfg2mac(u64 cfg, u8 *mac_addr)
 228 {
 229         int i, index = 0;
 230
 231         for (i = ETH_ALEN - 1; i >= 0; i--, index++)
 232                 mac_addr[i] = (cfg >> (8 * index)) & 0xFF;
 233 }
 234
 235 int cgx_lmac_addr_set(u8 cgx_id, u8 lmac_id, u8 *mac_addr)
 236 {
 237         struct cgx *cgx_dev = cgx_get_pdata(cgx_id);
 238         struct lmac *lmac = lmac_pdata(lmac_id, cgx_dev);
 239         struct mac_ops *mac_ops;
 240         int index, id;
 241         u64 cfg;
 242
 243         /* access mac_ops to know csr_offset */
 244         mac_ops = cgx_dev->mac_ops;
 245
 246         /* copy 6bytes from macaddr */
 247         /* memcpy(&cfg, mac_addr, 6); */
 248
 249         cfg = mac2u64 (mac_addr);
 250
 251         id = get_sequence_id_of_lmac(cgx_dev, lmac_id);
 252
 253         index = id * lmac->mac_to_index_bmap.max;
 254
 255         cgx_write(cgx_dev, 0, (CGXX_CMRX_RX_DMAC_CAM0 + (index * 0x8)),
 256                   cfg | CGX_DMAC_CAM_ADDR_ENABLE | ((u64)lmac_id << 49));
 257
 258         cfg = cgx_read(cgx_dev, lmac_id, CGXX_CMRX_RX_DMAC_CTL0);
 259         cfg |= (CGX_DMAC_CTL0_CAM_ENABLE | CGX_DMAC_BCAST_MODE |
 260                 CGX_DMAC_MCAST_MODE);
 261         cgx_write(cgx_dev, lmac_id, CGXX_CMRX_RX_DMAC_CTL0, cfg);
 262
 263         return 0;
 264 }
 265
 266 u64 cgx_read_dmac_ctrl(void *cgxd, int lmac_id)
 267 {
 268         struct mac_ops *mac_ops;
 269         struct cgx *cgx = cgxd;
 270
 271         if (!cgxd || !is_lmac_valid(cgxd, lmac_id))
 272                 return 0;
 273
 274         cgx = cgxd;
 275         /* Get mac_ops to know csr offset */
 276         mac_ops = cgx->mac_ops;
 277
 278         return cgx_read(cgxd, lmac_id, CGXX_CMRX_RX_DMAC_CTL0);
 279 }
 280
 281 u64 cgx_read_dmac_entry(void *cgxd, int index)
 282 {
 283         struct mac_ops *mac_ops;
 284         struct cgx *cgx;
 285
 286         if (!cgxd)
 287                 return 0;
 288
 289         cgx = cgxd;
 290         mac_ops = cgx->mac_ops;
 291         return cgx_read(cgx, 0, (CGXX_CMRX_RX_DMAC_CAM0 + (index * 8)));
 292 }
 293
 294 int cgx_lmac_addr_add(u8 cgx_id, u8 lmac_id, u8 *mac_addr)
 295 {
 296         struct cgx *cgx_dev = cgx_get_pdata(cgx_id);
 297         struct lmac *lmac = lmac_pdata(lmac_id, cgx_dev);
 298         struct mac_ops *mac_ops;
 299         int index, idx;
 300         u64 cfg = 0;
 301         int id;
 302
 303         if (!lmac)
 304                 return -ENODEV;
 305
 306         mac_ops = cgx_dev->mac_ops;
 307         /* Get available index where entry is to be installed */
 308         idx = rvu_alloc_rsrc(&lmac->mac_to_index_bmap);
 309         if (idx < 0)
 310                 return idx;
 311
 312         id = get_sequence_id_of_lmac(cgx_dev, lmac_id);
 313
 314         index = id * lmac->mac_to_index_bmap.max + idx;
 315
 316         cfg = mac2u64 (mac_addr);
 317         cfg |= CGX_DMAC_CAM_ADDR_ENABLE;
 318         cfg |= ((u64)lmac_id << 49);
 319         cgx_write(cgx_dev, 0, (CGXX_CMRX_RX_DMAC_CAM0 + (index * 0x8)), cfg);
 320
 321         cfg = cgx_read(cgx_dev, lmac_id, CGXX_CMRX_RX_DMAC_CTL0);
 322         cfg |= (CGX_DMAC_BCAST_MODE | CGX_DMAC_CAM_ACCEPT);
 323
 324         if (is_multicast_ether_addr(mac_addr)) {
 325                 cfg &= ~GENMASK_ULL(2, 1);
 326                 cfg |= CGX_DMAC_MCAST_MODE_CAM;
 327                 lmac->mcast_filters_count++;
 328         } else if (!lmac->mcast_filters_count) {
 329                 cfg |= CGX_DMAC_MCAST_MODE;
 330         }
 331
 332         cgx_write(cgx_dev, lmac_id, CGXX_CMRX_RX_DMAC_CTL0, cfg);
 333
 334         return idx;
 335 }
 336
 337 int cgx_lmac_addr_reset(u8 cgx_id, u8 lmac_id)
 338 {
 339         struct cgx *cgx_dev = cgx_get_pdata(cgx_id);
 340         struct lmac *lmac = lmac_pdata(lmac_id, cgx_dev);
 341         struct mac_ops *mac_ops;
 342         u8 index = 0, id;
 343         u64 cfg;
 344
 345         if (!lmac)
 346                 return -ENODEV;
 347
 348         mac_ops = cgx_dev->mac_ops;
 349         /* Restore index 0 to its default init value as done during
 350          * cgx_lmac_init
 351          */
 352         set_bit(0, lmac->mac_to_index_bmap.bmap);
 353
 354         id = get_sequence_id_of_lmac(cgx_dev, lmac_id);
 355
 356         index = id * lmac->mac_to_index_bmap.max + index;
 357         cgx_write(cgx_dev, 0, (CGXX_CMRX_RX_DMAC_CAM0 + (index * 0x8)), 0);
 358
 359         /* Reset CGXX_CMRX_RX_DMAC_CTL0 register to default state */
 360         cfg = cgx_read(cgx_dev, lmac_id, CGXX_CMRX_RX_DMAC_CTL0);
 361         cfg &= ~CGX_DMAC_CAM_ACCEPT;
 362         cfg |= (CGX_DMAC_BCAST_MODE | CGX_DMAC_MCAST_MODE);
 363         cgx_write(cgx_dev, lmac_id, CGXX_CMRX_RX_DMAC_CTL0, cfg);
 364
 365         return 0;
 366 }
 367
 368 /* Allows caller to change macaddress associated with index
 369  * in dmac filter table including index 0 reserved for
 370  * interface mac address
 371  */
 372 int cgx_lmac_addr_update(u8 cgx_id, u8 lmac_id, u8 *mac_addr, u8 index)
 373 {
 374         struct cgx *cgx_dev = cgx_get_pdata(cgx_id);
 375         struct mac_ops *mac_ops;
 376         struct lmac *lmac;
 377         u64 cfg;
 378         int id;
 379
 380         lmac = lmac_pdata(lmac_id, cgx_dev);
 381         if (!lmac)
 382                 return -ENODEV;
 383
 384         mac_ops = cgx_dev->mac_ops;
 385         /* Validate the index */
 386         if (index >= lmac->mac_to_index_bmap.max)
 387                 return -EINVAL;
 388
 389         /* ensure index is already set */
 390         if (!test_bit(index, lmac->mac_to_index_bmap.bmap))
 391                 return -EINVAL;
 392
 393         id = get_sequence_id_of_lmac(cgx_dev, lmac_id);
 394
 395         index = id * lmac->mac_to_index_bmap.max + index;
 396
 397         cfg = cgx_read(cgx_dev, 0, (CGXX_CMRX_RX_DMAC_CAM0 + (index * 0x8)));
 398         cfg &= ~CGX_RX_DMAC_ADR_MASK;
 399         cfg |= mac2u64 (mac_addr);
 400
 401         cgx_write(cgx_dev, 0, (CGXX_CMRX_RX_DMAC_CAM0 + (index * 0x8)), cfg);
 402         return 0;
 403 }
 404
 405 int cgx_lmac_addr_del(u8 cgx_id, u8 lmac_id, u8 index)
 406 {
 407         struct cgx *cgx_dev = cgx_get_pdata(cgx_id);
 408         struct lmac *lmac = lmac_pdata(lmac_id, cgx_dev);
 409         struct mac_ops *mac_ops;
 410         u8 mac[ETH_ALEN];
 411         u64 cfg;
 412         int id;
 413
 414         if (!lmac)
 415                 return -ENODEV;
 416
 417         mac_ops = cgx_dev->mac_ops;
 418         /* Validate the index */
 419         if (index >= lmac->mac_to_index_bmap.max)
 420                 return -EINVAL;
 421
 422         /* Skip deletion for reserved index i.e. index 0 */
 423         if (index == 0)
 424                 return 0;
 425
 426         rvu_free_rsrc(&lmac->mac_to_index_bmap, index);
 427
 428         id = get_sequence_id_of_lmac(cgx_dev, lmac_id);
 429
 430         index = id * lmac->mac_to_index_bmap.max + index;
 431
 432         /* Read MAC address to check whether it is ucast or mcast */
 433         cfg = cgx_read(cgx_dev, 0, (CGXX_CMRX_RX_DMAC_CAM0 + (index * 0x8)));
 434
 435         cfg2mac(cfg, mac);
 436         if (is_multicast_ether_addr(mac))
 437                 lmac->mcast_filters_count--;
 438
 439         if (!lmac->mcast_filters_count) {
 440                 cfg = cgx_read(cgx_dev, lmac_id, CGXX_CMRX_RX_DMAC_CTL0);
 441                 cfg &= ~GENMASK_ULL(2, 1);
 442                 cfg |= CGX_DMAC_MCAST_MODE;
 443                 cgx_write(cgx_dev, lmac_id, CGXX_CMRX_RX_DMAC_CTL0, cfg);
 444         }
 445
 446         cgx_write(cgx_dev, 0, (CGXX_CMRX_RX_DMAC_CAM0 + (index * 0x8)), 0);
 447
 448         return 0;
 449 }
 450
 451 int cgx_lmac_addr_max_entries_get(u8 cgx_id, u8 lmac_id)
 452 {
 453         struct cgx *cgx_dev = cgx_get_pdata(cgx_id);
 454         struct lmac *lmac = lmac_pdata(lmac_id, cgx_dev);
 455
 456         if (lmac)
 457                 return lmac->mac_to_index_bmap.max;
 458
 459         return 0;
 460 }
 461
 462 u64 cgx_lmac_addr_get(u8 cgx_id, u8 lmac_id)
 463 {
 464         struct cgx *cgx_dev = cgx_get_pdata(cgx_id);
 465         struct lmac *lmac = lmac_pdata(lmac_id, cgx_dev);
 466         struct mac_ops *mac_ops;
 467         int index;
 468         u64 cfg;
 469         int id;
 470
 471         mac_ops = cgx_dev->mac_ops;
 472
 473         id = get_sequence_id_of_lmac(cgx_dev, lmac_id);
 474
 475         index = id * lmac->mac_to_index_bmap.max;
 476
 477         cfg = cgx_read(cgx_dev, 0, CGXX_CMRX_RX_DMAC_CAM0 + index * 0x8);
 478         return cfg & CGX_RX_DMAC_ADR_MASK;
 479 }
 480
 481 int cgx_set_pkind(void *cgxd, u8 lmac_id, int pkind)
 482 {
 483         struct cgx *cgx = cgxd;
 484
 485         if (!is_lmac_valid(cgx, lmac_id))
 486                 return -ENODEV;
 487
 488         cgx_write(cgx, lmac_id, CGXX_CMRX_RX_ID_MAP, (pkind & 0x3F));
 489         return 0;
 490 }
 491
 492 static u8 cgx_get_lmac_type(void *cgxd, int lmac_id)
 493 {
 494         struct cgx *cgx = cgxd;
 495         u64 cfg;
 496
 497         cfg = cgx_read(cgx, lmac_id, CGXX_CMRX_CFG);
 498         return (cfg >> CGX_LMAC_TYPE_SHIFT) & CGX_LMAC_TYPE_MASK;
 499 }
 500
 501 /* Configure CGX LMAC in internal loopback mode */
 502 int cgx_lmac_internal_loopback(void *cgxd, int lmac_id, bool enable)
 503 {
 504         struct cgx *cgx = cgxd;
 505         u8 lmac_type;
 506         u64 cfg;
 507
 508         if (!is_lmac_valid(cgx, lmac_id))
 509                 return -ENODEV;
 510
 511         lmac_type = cgx->mac_ops->get_lmac_type(cgx, lmac_id);
 512         if (lmac_type == LMAC_MODE_SGMII || lmac_type == LMAC_MODE_QSGMII) {
 513                 cfg = cgx_read(cgx, lmac_id, CGXX_GMP_PCS_MRX_CTL);
 514                 if (enable)
 515                         cfg |= CGXX_GMP_PCS_MRX_CTL_LBK;
 516                 else
 517                         cfg &= ~CGXX_GMP_PCS_MRX_CTL_LBK;
 518                 cgx_write(cgx, lmac_id, CGXX_GMP_PCS_MRX_CTL, cfg);
 519         } else {
 520                 cfg = cgx_read(cgx, lmac_id, CGXX_SPUX_CONTROL1);
 521                 if (enable)
 522                         cfg |= CGXX_SPUX_CONTROL1_LBK;
 523                 else
 524                         cfg &= ~CGXX_SPUX_CONTROL1_LBK;
 525                 cgx_write(cgx, lmac_id, CGXX_SPUX_CONTROL1, cfg);
 526         }
 527         return 0;
 528 }
 529
 530 void cgx_lmac_promisc_config(int cgx_id, int lmac_id, bool enable)
 531 {
 532         struct cgx *cgx = cgx_get_pdata(cgx_id);
 533         struct lmac *lmac = lmac_pdata(lmac_id, cgx);
 534         u16 max_dmac = lmac->mac_to_index_bmap.max;
 535         struct mac_ops *mac_ops;
 536         int index, i;
 537         u64 cfg = 0;
 538         int id;
 539
 540         if (!cgx)
 541                 return;
 542
 543         id = get_sequence_id_of_lmac(cgx, lmac_id);
 544
 545         mac_ops = cgx->mac_ops;
 546         if (enable) {
 547                 /* Enable promiscuous mode on LMAC */
 548                 cfg = cgx_read(cgx, lmac_id, CGXX_CMRX_RX_DMAC_CTL0);
 549                 cfg &= ~CGX_DMAC_CAM_ACCEPT;
 550                 cfg |= (CGX_DMAC_BCAST_MODE | CGX_DMAC_MCAST_MODE);
 551                 cgx_write(cgx, lmac_id, CGXX_CMRX_RX_DMAC_CTL0, cfg);
 552
 553                 for (i = 0; i < max_dmac; i++) {
 554                         index = id * max_dmac + i;
 555                         cfg = cgx_read(cgx, 0,
 556                                        (CGXX_CMRX_RX_DMAC_CAM0 + index * 0x8));
 557                         cfg &= ~CGX_DMAC_CAM_ADDR_ENABLE;
 558                         cgx_write(cgx, 0,
 559                                   (CGXX_CMRX_RX_DMAC_CAM0 + index * 0x8), cfg);
 560                 }
 561         } else {
 562                 /* Disable promiscuous mode */
 563                 cfg = cgx_read(cgx, lmac_id, CGXX_CMRX_RX_DMAC_CTL0);
 564                 cfg |= CGX_DMAC_CAM_ACCEPT | CGX_DMAC_MCAST_MODE;
 565                 cgx_write(cgx, lmac_id, CGXX_CMRX_RX_DMAC_CTL0, cfg);
 566                 for (i = 0; i < max_dmac; i++) {
 567                         index = id * max_dmac + i;
 568                         cfg = cgx_read(cgx, 0,
 569                                        (CGXX_CMRX_RX_DMAC_CAM0 + index * 0x8));
 570                         if ((cfg & CGX_RX_DMAC_ADR_MASK) != 0) {
 571                                 cfg |= CGX_DMAC_CAM_ADDR_ENABLE;
 572                                 cgx_write(cgx, 0,
 573                                           (CGXX_CMRX_RX_DMAC_CAM0 +
 574                                            index * 0x8),
 575                                           cfg);
 576                         }
 577                 }
 578         }
 579 }
 580
 581 /* Enable or disable forwarding received pause frames to Tx block */
 582 void cgx_lmac_enadis_rx_pause_fwding(void *cgxd, int lmac_id, bool enable)
 583 {
 584         struct cgx *cgx = cgxd;
 585         u64 cfg;
 586
 587         if (!cgx)
 588                 return;
 589
 590         if (enable) {
 591                 cfg = cgx_read(cgx, lmac_id, CGXX_GMP_GMI_RXX_FRM_CTL);
 592                 cfg |= CGX_GMP_GMI_RXX_FRM_CTL_CTL_BCK;
 593                 cgx_write(cgx, lmac_id, CGXX_GMP_GMI_RXX_FRM_CTL, cfg);
 594
 595                 cfg = cgx_read(cgx, lmac_id, CGXX_SMUX_RX_FRM_CTL);
 596                 cfg |= CGX_SMUX_RX_FRM_CTL_CTL_BCK;
 597                 cgx_write(cgx, lmac_id, CGXX_SMUX_RX_FRM_CTL, cfg);
 598         } else {
 599                 cfg = cgx_read(cgx, lmac_id, CGXX_GMP_GMI_RXX_FRM_CTL);
 600                 cfg &= ~CGX_GMP_GMI_RXX_FRM_CTL_CTL_BCK;
 601                 cgx_write(cgx, lmac_id, CGXX_GMP_GMI_RXX_FRM_CTL, cfg);
 602
 603                 cfg = cgx_read(cgx, lmac_id, CGXX_SMUX_RX_FRM_CTL);
 604                 cfg &= ~CGX_SMUX_RX_FRM_CTL_CTL_BCK;
 605                 cgx_write(cgx, lmac_id, CGXX_SMUX_RX_FRM_CTL, cfg);
 606         }
 607 }
 608
 609 int cgx_get_rx_stats(void *cgxd, int lmac_id, int idx, u64 *rx_stat)
 610 {
 611         struct cgx *cgx = cgxd;
 612
 613         if (!is_lmac_valid(cgx, lmac_id))
 614                 return -ENODEV;
 615         *rx_stat =  cgx_read(cgx, lmac_id, CGXX_CMRX_RX_STAT0 + (idx * 8));
 616         return 0;
 617 }
 618
 619 int cgx_get_tx_stats(void *cgxd, int lmac_id, int idx, u64 *tx_stat)
 620 {
 621         struct cgx *cgx = cgxd;
 622
 623         if (!is_lmac_valid(cgx, lmac_id))
 624                 return -ENODEV;
 625         *tx_stat = cgx_read(cgx, lmac_id, CGXX_CMRX_TX_STAT0 + (idx * 8));
 626         return 0;
 627 }
 628
 629 u64 cgx_features_get(void *cgxd)
 630 {
 631         return ((struct cgx *)cgxd)->hw_features;
 632 }
 633
 634 static int cgx_set_fec_stats_count(struct cgx_link_user_info *linfo)
 635 {
 636         if (!linfo->fec)
 637                 return 0;
 638
 639         switch (linfo->lmac_type_id) {
 640         case LMAC_MODE_SGMII:
 641         case LMAC_MODE_XAUI:
 642         case LMAC_MODE_RXAUI:
 643         case LMAC_MODE_QSGMII:
 644                 return 0;
 645         case LMAC_MODE_10G_R:
 646         case LMAC_MODE_25G_R:
 647         case LMAC_MODE_100G_R:
 648         case LMAC_MODE_USXGMII:
 649                 return 1;
 650         case LMAC_MODE_40G_R:
 651                 return 4;
 652         case LMAC_MODE_50G_R:
 653                 if (linfo->fec == OTX2_FEC_BASER)
 654                         return 2;
 655                 else
 656                         return 1;
 657         default:
 658                 return 0;
 659         }
 660 }
 661
 662 int cgx_get_fec_stats(void *cgxd, int lmac_id, struct cgx_fec_stats_rsp *rsp)
 663 {
 664         int stats, fec_stats_count = 0;
 665         int corr_reg, uncorr_reg;
 666         struct cgx *cgx = cgxd;
 667
 668         if (!cgx || lmac_id >= cgx->lmac_count)
 669                 return -ENODEV;
 670         fec_stats_count =
 671                 cgx_set_fec_stats_count(&cgx->lmac_idmap[lmac_id]->link_info);
 672         if (cgx->lmac_idmap[lmac_id]->link_info.fec == OTX2_FEC_BASER) {
 673                 corr_reg = CGXX_SPUX_LNX_FEC_CORR_BLOCKS;
 674                 uncorr_reg = CGXX_SPUX_LNX_FEC_UNCORR_BLOCKS;
 675         } else {
 676                 corr_reg = CGXX_SPUX_RSFEC_CORR;
 677                 uncorr_reg = CGXX_SPUX_RSFEC_UNCORR;
 678         }
 679         for (stats = 0; stats < fec_stats_count; stats++) {
 680                 rsp->fec_corr_blks +=
 681                         cgx_read(cgx, lmac_id, corr_reg + (stats * 8));
 682                 rsp->fec_uncorr_blks +=
 683                         cgx_read(cgx, lmac_id, uncorr_reg + (stats * 8));
 684         }
 685         return 0;
 686 }
 687
 688 int cgx_lmac_rx_tx_enable(void *cgxd, int lmac_id, bool enable)
 689 {
 690         struct cgx *cgx = cgxd;
 691         u64 cfg;
 692
 693         if (!is_lmac_valid(cgx, lmac_id))
 694                 return -ENODEV;
 695
 696         cfg = cgx_read(cgx, lmac_id, CGXX_CMRX_CFG);
 697         if (enable)
 698                 cfg |= CMR_EN | DATA_PKT_RX_EN | DATA_PKT_TX_EN;
 699         else
 700                 cfg &= ~(CMR_EN | DATA_PKT_RX_EN | DATA_PKT_TX_EN);
 701         cgx_write(cgx, lmac_id, CGXX_CMRX_CFG, cfg);
 702         return 0;
 703 }
 704
 705 int cgx_lmac_tx_enable(void *cgxd, int lmac_id, bool enable)
 706 {
 707         struct cgx *cgx = cgxd;
 708         u64 cfg, last;
 709
 710         if (!is_lmac_valid(cgx, lmac_id))
 711                 return -ENODEV;
 712
 713         cfg = cgx_read(cgx, lmac_id, CGXX_CMRX_CFG);
 714         last = cfg;
 715         if (enable)
 716                 cfg |= DATA_PKT_TX_EN;
 717         else
 718                 cfg &= ~DATA_PKT_TX_EN;
 719
 720         if (cfg != last)
 721                 cgx_write(cgx, lmac_id, CGXX_CMRX_CFG, cfg);
 722         return !!(last & DATA_PKT_TX_EN);
 723 }
 724
 725 static int cgx_lmac_get_pause_frm_status(void *cgxd, int lmac_id,
 726                                          u8 *tx_pause, u8 *rx_pause)
 727 {
 728         struct cgx *cgx = cgxd;
 729         u64 cfg;
 730
 731         if (is_dev_rpm(cgx))
 732                 return 0;
 733
 734         if (!is_lmac_valid(cgx, lmac_id))
 735                 return -ENODEV;
 736
 737         cfg = cgx_read(cgx, lmac_id, CGXX_SMUX_RX_FRM_CTL);
 738         *rx_pause = !!(cfg & CGX_SMUX_RX_FRM_CTL_CTL_BCK);
 739
 740         cfg = cgx_read(cgx, lmac_id, CGXX_SMUX_TX_CTL);
 741         *tx_pause = !!(cfg & CGX_SMUX_TX_CTL_L2P_BP_CONV);
 742         return 0;
 743 }
 744
 745 static int cgx_lmac_enadis_pause_frm(void *cgxd, int lmac_id,
 746                                      u8 tx_pause, u8 rx_pause)
 747 {
 748         struct cgx *cgx = cgxd;
 749         u64 cfg;
 750
 751         if (is_dev_rpm(cgx))
 752                 return 0;
 753
 754         if (!is_lmac_valid(cgx, lmac_id))
 755                 return -ENODEV;
 756
 757         cfg = cgx_read(cgx, lmac_id, CGXX_SMUX_RX_FRM_CTL);
 758         cfg &= ~CGX_SMUX_RX_FRM_CTL_CTL_BCK;
 759         cfg |= rx_pause ? CGX_SMUX_RX_FRM_CTL_CTL_BCK : 0x0;
 760         cgx_write(cgx, lmac_id, CGXX_SMUX_RX_FRM_CTL, cfg);
 761
 762         cfg = cgx_read(cgx, lmac_id, CGXX_SMUX_TX_CTL);
 763         cfg &= ~CGX_SMUX_TX_CTL_L2P_BP_CONV;
 764         cfg |= tx_pause ? CGX_SMUX_TX_CTL_L2P_BP_CONV : 0x0;
 765         cgx_write(cgx, lmac_id, CGXX_SMUX_TX_CTL, cfg);
 766
 767         cfg = cgx_read(cgx, 0, CGXX_CMR_RX_OVR_BP);
 768         if (tx_pause) {
 769                 cfg &= ~CGX_CMR_RX_OVR_BP_EN(lmac_id);
 770         } else {
 771                 cfg |= CGX_CMR_RX_OVR_BP_EN(lmac_id);
 772                 cfg &= ~CGX_CMR_RX_OVR_BP_BP(lmac_id);
 773         }
 774         cgx_write(cgx, 0, CGXX_CMR_RX_OVR_BP, cfg);
 775         return 0;
 776 }
 777
 778 static void cgx_lmac_pause_frm_config(void *cgxd, int lmac_id, bool enable)
 779 {
 780         struct cgx *cgx = cgxd;
 781         u64 cfg;
 782
 783         if (!is_lmac_valid(cgx, lmac_id))
 784                 return;
 785         if (enable) {
 786                 /* Enable receive pause frames */
 787                 cfg = cgx_read(cgx, lmac_id, CGXX_SMUX_RX_FRM_CTL);
 788                 cfg |= CGX_SMUX_RX_FRM_CTL_CTL_BCK;
 789                 cgx_write(cgx, lmac_id, CGXX_SMUX_RX_FRM_CTL, cfg);
 790
 791                 cfg = cgx_read(cgx, lmac_id, CGXX_GMP_GMI_RXX_FRM_CTL);
 792                 cfg |= CGX_GMP_GMI_RXX_FRM_CTL_CTL_BCK;
 793                 cgx_write(cgx, lmac_id, CGXX_GMP_GMI_RXX_FRM_CTL, cfg);
 794
 795                 /* Enable pause frames transmission */
 796                 cfg = cgx_read(cgx, lmac_id, CGXX_SMUX_TX_CTL);
 797                 cfg |= CGX_SMUX_TX_CTL_L2P_BP_CONV;
 798                 cgx_write(cgx, lmac_id, CGXX_SMUX_TX_CTL, cfg);
 799
 800                 /* Set pause time and interval */
 801                 cgx_write(cgx, lmac_id, CGXX_SMUX_TX_PAUSE_PKT_TIME,
 802                           DEFAULT_PAUSE_TIME);
 803                 cfg = cgx_read(cgx, lmac_id, CGXX_SMUX_TX_PAUSE_PKT_INTERVAL);
 804                 cfg &= ~0xFFFFULL;
 805                 cgx_write(cgx, lmac_id, CGXX_SMUX_TX_PAUSE_PKT_INTERVAL,
 806                           cfg | (DEFAULT_PAUSE_TIME / 2));
 807
 808                 cgx_write(cgx, lmac_id, CGXX_GMP_GMI_TX_PAUSE_PKT_TIME,
 809                           DEFAULT_PAUSE_TIME);
 810
 811                 cfg = cgx_read(cgx, lmac_id,
 812                                CGXX_GMP_GMI_TX_PAUSE_PKT_INTERVAL);
 813                 cfg &= ~0xFFFFULL;
 814                 cgx_write(cgx, lmac_id, CGXX_GMP_GMI_TX_PAUSE_PKT_INTERVAL,
 815                           cfg | (DEFAULT_PAUSE_TIME / 2));
 816         } else {
 817                 /* ALL pause frames received are completely ignored */
 818                 cfg = cgx_read(cgx, lmac_id, CGXX_SMUX_RX_FRM_CTL);
 819                 cfg &= ~CGX_SMUX_RX_FRM_CTL_CTL_BCK;
 820                 cgx_write(cgx, lmac_id, CGXX_SMUX_RX_FRM_CTL, cfg);
 821
 822                 cfg = cgx_read(cgx, lmac_id, CGXX_GMP_GMI_RXX_FRM_CTL);
 823                 cfg &= ~CGX_GMP_GMI_RXX_FRM_CTL_CTL_BCK;
 824                 cgx_write(cgx, lmac_id, CGXX_GMP_GMI_RXX_FRM_CTL, cfg);
 825
 826                 /* Disable pause frames transmission */
 827                 cfg = cgx_read(cgx, lmac_id, CGXX_SMUX_TX_CTL);
 828                 cfg &= ~CGX_SMUX_TX_CTL_L2P_BP_CONV;
 829                 cgx_write(cgx, lmac_id, CGXX_SMUX_TX_CTL, cfg);
 830         }
 831 }
 832
 833 void cgx_lmac_ptp_config(void *cgxd, int lmac_id, bool enable)
 834 {
 835         struct cgx *cgx = cgxd;
 836         u64 cfg;
 837
 838         if (!cgx)
 839                 return;
 840
 841         if (enable) {
 842                 /* Enable inbound PTP timestamping */
 843                 cfg = cgx_read(cgx, lmac_id, CGXX_GMP_GMI_RXX_FRM_CTL);
 844                 cfg |= CGX_GMP_GMI_RXX_FRM_CTL_PTP_MODE;
 845                 cgx_write(cgx, lmac_id, CGXX_GMP_GMI_RXX_FRM_CTL, cfg);
 846
 847                 cfg = cgx_read(cgx, lmac_id, CGXX_SMUX_RX_FRM_CTL);
 848                 cfg |= CGX_SMUX_RX_FRM_CTL_PTP_MODE;
 849                 cgx_write(cgx, lmac_id, CGXX_SMUX_RX_FRM_CTL, cfg);
 850         } else {
 851                 /* Disable inbound PTP stamping */
 852                 cfg = cgx_read(cgx, lmac_id, CGXX_GMP_GMI_RXX_FRM_CTL);
 853                 cfg &= ~CGX_GMP_GMI_RXX_FRM_CTL_PTP_MODE;
 854                 cgx_write(cgx, lmac_id, CGXX_GMP_GMI_RXX_FRM_CTL, cfg);
 855
 856                 cfg = cgx_read(cgx, lmac_id, CGXX_SMUX_RX_FRM_CTL);
 857                 cfg &= ~CGX_SMUX_RX_FRM_CTL_PTP_MODE;
 858                 cgx_write(cgx, lmac_id, CGXX_SMUX_RX_FRM_CTL, cfg);
 859         }
 860 }
 861
 862 /* CGX Firmware interface low level support */
 863 int cgx_fwi_cmd_send(u64 req, u64 *resp, struct lmac *lmac)
 864 {
 865         struct cgx *cgx = lmac->cgx;
 866         struct device *dev;
 867         int err = 0;
 868         u64 cmd;
 869
 870         /* Ensure no other command is in progress */
 871         err = mutex_lock_interruptible(&lmac->cmd_lock);
 872         if (err)
 873                 return err;
 874
 875         /* Ensure command register is free */
 876         cmd = cgx_read(cgx, lmac->lmac_id,  CGX_COMMAND_REG);
 877         if (FIELD_GET(CMDREG_OWN, cmd) != CGX_CMD_OWN_NS) {
 878                 err = -EBUSY;
 879                 goto unlock;
 880         }
 881
 882         /* Update ownership in command request */
 883         req = FIELD_SET(CMDREG_OWN, CGX_CMD_OWN_FIRMWARE, req);
 884
 885         /* Mark this lmac as pending, before we start */
 886         lmac->cmd_pend = true;
 887
 888         /* Start command in hardware */
 889         cgx_write(cgx, lmac->lmac_id, CGX_COMMAND_REG, req);
 890
 891         /* Ensure command is completed without errors */
 892         if (!wait_event_timeout(lmac->wq_cmd_cmplt, !lmac->cmd_pend,
 893                                 msecs_to_jiffies(CGX_CMD_TIMEOUT))) {
 894                 dev = &cgx->pdev->dev;
 895                 dev_err(dev, "cgx port %d:%d cmd timeout\n",
 896                         cgx->cgx_id, lmac->lmac_id);
 897                 err = -EIO;
 898                 goto unlock;
 899         }
 900
 901         /* we have a valid command response */
 902         smp_rmb(); /* Ensure the latest updates are visible */
 903         *resp = lmac->resp;
 904
 905 unlock:
 906         mutex_unlock(&lmac->cmd_lock);
 907
 908         return err;
 909 }
 910
 911 int cgx_fwi_cmd_generic(u64 req, u64 *resp, struct cgx *cgx, int lmac_id)
 912 {
 913         struct lmac *lmac;
 914         int err;
 915
 916         lmac = lmac_pdata(lmac_id, cgx);
 917         if (!lmac)
 918                 return -ENODEV;
 919
 920         err = cgx_fwi_cmd_send(req, resp, lmac);
 921
 922         /* Check for valid response */
 923         if (!err) {
 924                 if (FIELD_GET(EVTREG_STAT, *resp) == CGX_STAT_FAIL)
 925                         return -EIO;
 926                 else
 927                         return 0;
 928         }
 929
 930         return err;
 931 }
 932
 933 static int cgx_link_usertable_index_map(int speed)
 934 {
 935         switch (speed) {
 936         case SPEED_10:
 937                 return CGX_LINK_10M;
 938         case SPEED_100:
 939                 return CGX_LINK_100M;
 940         case SPEED_1000:
 941                 return CGX_LINK_1G;
 942         case SPEED_2500:
 943                 return CGX_LINK_2HG;
 944         case SPEED_5000:
 945                 return CGX_LINK_5G;
 946         case SPEED_10000:
 947                 return CGX_LINK_10G;
 948         case SPEED_20000:
 949                 return CGX_LINK_20G;
 950         case SPEED_25000:
 951                 return CGX_LINK_25G;
 952         case SPEED_40000:
 953                 return CGX_LINK_40G;
 954         case SPEED_50000:
 955                 return CGX_LINK_50G;
 956         case 80000:
 957                 return CGX_LINK_80G;
 958         case SPEED_100000:
 959                 return CGX_LINK_100G;
 960         case SPEED_UNKNOWN:
 961                 return CGX_LINK_NONE;
 962         }
 963         return CGX_LINK_NONE;
 964 }
 965
 966 static void set_mod_args(struct cgx_set_link_mode_args *args,
 967                          u32 speed, u8 duplex, u8 autoneg, u64 mode)
 968 {
 969         /* Fill default values incase of user did not pass
 970          * valid parameters
 971          */
 972         if (args->duplex == DUPLEX_UNKNOWN)
 973                 args->duplex = duplex;
 974         if (args->speed == SPEED_UNKNOWN)
 975                 args->speed = speed;
 976         if (args->an == AUTONEG_UNKNOWN)
 977                 args->an = autoneg;
 978         args->mode = mode;
 979         args->ports = 0;
 980 }
 981
 982 static void otx2_map_ethtool_link_modes(u64 bitmask,
 983                                         struct cgx_set_link_mode_args *args)
 984 {
 985         switch (bitmask) {
 986         case ETHTOOL_LINK_MODE_10baseT_Half_BIT:
 987                 set_mod_args(args, 10, 1, 1, BIT_ULL(CGX_MODE_SGMII));
 988                 break;
 989         case  ETHTOOL_LINK_MODE_10baseT_Full_BIT:
 990                 set_mod_args(args, 10, 0, 1, BIT_ULL(CGX_MODE_SGMII));
 991                 break;
 992         case  ETHTOOL_LINK_MODE_100baseT_Half_BIT:
 993                 set_mod_args(args, 100, 1, 1, BIT_ULL(CGX_MODE_SGMII));
 994                 break;
 995         case  ETHTOOL_LINK_MODE_100baseT_Full_BIT:
 996                 set_mod_args(args, 100, 0, 1, BIT_ULL(CGX_MODE_SGMII));
 997                 break;
 998         case  ETHTOOL_LINK_MODE_1000baseT_Half_BIT:
 999                 set_mod_args(args, 1000, 1, 1, BIT_ULL(CGX_MODE_SGMII));
1000                 break;
1001         case  ETHTOOL_LINK_MODE_1000baseT_Full_BIT:
1002                 set_mod_args(args, 1000, 0, 1, BIT_ULL(CGX_MODE_SGMII));
1003                 break;
1004         case  ETHTOOL_LINK_MODE_1000baseX_Full_BIT:
1005                 set_mod_args(args, 1000, 0, 0, BIT_ULL(CGX_MODE_1000_BASEX));
1006                 break;
1007         case  ETHTOOL_LINK_MODE_10000baseT_Full_BIT:
1008                 set_mod_args(args, 1000, 0, 1, BIT_ULL(CGX_MODE_QSGMII));
1009                 break;
1010         case  ETHTOOL_LINK_MODE_10000baseSR_Full_BIT:
1011                 set_mod_args(args, 10000, 0, 0, BIT_ULL(CGX_MODE_10G_C2C));
1012                 break;
1013         case  ETHTOOL_LINK_MODE_10000baseLR_Full_BIT:
1014                 set_mod_args(args, 10000, 0, 0, BIT_ULL(CGX_MODE_10G_C2M));
1015                 break;
1016         case  ETHTOOL_LINK_MODE_10000baseKR_Full_BIT:
1017                 set_mod_args(args, 10000, 0, 1, BIT_ULL(CGX_MODE_10G_KR));
1018                 break;
1019         case  ETHTOOL_LINK_MODE_25000baseSR_Full_BIT:
1020                 set_mod_args(args, 25000, 0, 0, BIT_ULL(CGX_MODE_25G_C2C));
1021                 break;
1022         case  ETHTOOL_LINK_MODE_25000baseCR_Full_BIT:
1023                 set_mod_args(args, 25000, 0, 1, BIT_ULL(CGX_MODE_25G_CR));
1024                 break;
1025         case  ETHTOOL_LINK_MODE_25000baseKR_Full_BIT:
1026                 set_mod_args(args, 25000, 0, 1, BIT_ULL(CGX_MODE_25G_KR));
1027                 break;
1028         case  ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT:
1029                 set_mod_args(args, 40000, 0, 0, BIT_ULL(CGX_MODE_40G_C2C));
1030                 break;
1031         case  ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT:
1032                 set_mod_args(args, 40000, 0, 0, BIT_ULL(CGX_MODE_40G_C2M));
1033                 break;
1034         case  ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT:
1035                 set_mod_args(args, 40000, 0, 1, BIT_ULL(CGX_MODE_40G_CR4));
1036                 break;
1037         case  ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT:
1038                 set_mod_args(args, 40000, 0, 1, BIT_ULL(CGX_MODE_40G_KR4));
1039                 break;
1040         case  ETHTOOL_LINK_MODE_50000baseSR_Full_BIT:
1041                 set_mod_args(args, 50000, 0, 0, BIT_ULL(CGX_MODE_50G_C2C));
1042                 break;
1043         case  ETHTOOL_LINK_MODE_50000baseLR_ER_FR_Full_BIT:
1044                 set_mod_args(args, 50000, 0, 0, BIT_ULL(CGX_MODE_50G_C2M));
1045                 break;
1046         case  ETHTOOL_LINK_MODE_50000baseCR_Full_BIT:
1047                 set_mod_args(args, 50000, 0, 1, BIT_ULL(CGX_MODE_50G_CR));
1048                 break;
1049         case  ETHTOOL_LINK_MODE_50000baseKR_Full_BIT:
1050                 set_mod_args(args, 50000, 0, 1, BIT_ULL(CGX_MODE_50G_KR));
1051                 break;
1052         case  ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT:
1053                 set_mod_args(args, 100000, 0, 0, BIT_ULL(CGX_MODE_100G_C2C));
1054                 break;
1055         case  ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT:
1056                 set_mod_args(args, 100000, 0, 0, BIT_ULL(CGX_MODE_100G_C2M));
1057                 break;
1058         case  ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT:
1059                 set_mod_args(args, 100000, 0, 1, BIT_ULL(CGX_MODE_100G_CR4));
1060                 break;
1061         case  ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT:
1062                 set_mod_args(args, 100000, 0, 1, BIT_ULL(CGX_MODE_100G_KR4));
1063                 break;
1064         default:
1065                 set_mod_args(args, 0, 1, 0, BIT_ULL(CGX_MODE_MAX));
1066                 break;
1067         }
1068 }
1069
1070 static inline void link_status_user_format(u64 lstat,
1071                                            struct cgx_link_user_info *linfo,
1072                                            struct cgx *cgx, u8 lmac_id)
1073 {
1074         const char *lmac_string;
1075
1076         linfo->link_up = FIELD_GET(RESP_LINKSTAT_UP, lstat);
1077         linfo->full_duplex = FIELD_GET(RESP_LINKSTAT_FDUPLEX, lstat);
1078         linfo->speed = cgx_speed_mbps[FIELD_GET(RESP_LINKSTAT_SPEED, lstat)];
1079         linfo->an = FIELD_GET(RESP_LINKSTAT_AN, lstat);
1080         linfo->fec = FIELD_GET(RESP_LINKSTAT_FEC, lstat);
1081         linfo->lmac_type_id = cgx_get_lmac_type(cgx, lmac_id);
1082         lmac_string = cgx_lmactype_string[linfo->lmac_type_id];
1083         strncpy(linfo->lmac_type, lmac_string, LMACTYPE_STR_LEN - 1);
1084 }
1085
1086 /* Hardware event handlers */
1087 static inline void cgx_link_change_handler(u64 lstat,
1088                                            struct lmac *lmac)
1089 {
1090         struct cgx_link_user_info *linfo;
1091         struct cgx *cgx = lmac->cgx;
1092         struct cgx_link_event event;
1093         struct device *dev;
1094         int err_type;
1095
1096         dev = &cgx->pdev->dev;
1097
1098         link_status_user_format(lstat, &event.link_uinfo, cgx, lmac->lmac_id);
1099         err_type = FIELD_GET(RESP_LINKSTAT_ERRTYPE, lstat);
1100
1101         event.cgx_id = cgx->cgx_id;
1102         event.lmac_id = lmac->lmac_id;
1103
1104         /* update the local copy of link status */
1105         lmac->link_info = event.link_uinfo;
1106         linfo = &lmac->link_info;
1107
1108         if (err_type == CGX_ERR_SPEED_CHANGE_INVALID)
1109                 return;
1110
1111         /* Ensure callback doesn't get unregistered until we finish it */
1112         spin_lock(&lmac->event_cb_lock);
1113
1114         if (!lmac->event_cb.notify_link_chg) {
1115                 dev_dbg(dev, "cgx port %d:%d Link change handler null",
1116                         cgx->cgx_id, lmac->lmac_id);
1117                 if (err_type != CGX_ERR_NONE) {
1118                         dev_err(dev, "cgx port %d:%d Link error %d\n",
1119                                 cgx->cgx_id, lmac->lmac_id, err_type);
1120                 }
1121                 dev_info(dev, "cgx port %d:%d Link is %s %d Mbps\n",
1122                          cgx->cgx_id, lmac->lmac_id,
1123                          linfo->link_up ? "UP" : "DOWN", linfo->speed);
1124                 goto err;
1125         }
1126
1127         if (lmac->event_cb.notify_link_chg(&event, lmac->event_cb.data))
1128                 dev_err(dev, "event notification failure\n");
1129 err:
1130         spin_unlock(&lmac->event_cb_lock);
1131 }
1132
1133 static inline bool cgx_cmdresp_is_linkevent(u64 event)
1134 {
1135         u8 id;
1136
1137         id = FIELD_GET(EVTREG_ID, event);
1138         if (id == CGX_CMD_LINK_BRING_UP ||
1139             id == CGX_CMD_LINK_BRING_DOWN ||
1140             id == CGX_CMD_MODE_CHANGE)
1141                 return true;
1142         else
1143                 return false;
1144 }
1145
1146 static inline bool cgx_event_is_linkevent(u64 event)
1147 {
1148         if (FIELD_GET(EVTREG_ID, event) == CGX_EVT_LINK_CHANGE)
1149                 return true;
1150         else
1151                 return false;
1152 }
1153
1154 static irqreturn_t cgx_fwi_event_handler(int irq, void *data)
1155 {
1156         u64 event, offset, clear_bit;
1157         struct lmac *lmac = data;
1158         struct cgx *cgx;
1159
1160         cgx = lmac->cgx;
1161
1162         /* Clear SW_INT for RPM and CMR_INT for CGX */
1163         offset     = cgx->mac_ops->int_register;
1164         clear_bit  = cgx->mac_ops->int_ena_bit;
1165
1166         event = cgx_read(cgx, lmac->lmac_id, CGX_EVENT_REG);
1167
1168         if (!FIELD_GET(EVTREG_ACK, event))
1169                 return IRQ_NONE;
1170
1171         switch (FIELD_GET(EVTREG_EVT_TYPE, event)) {
1172         case CGX_EVT_CMD_RESP:
1173                 /* Copy the response. Since only one command is active at a
1174                  * time, there is no way a response can get overwritten
1175                  */
1176                 lmac->resp = event;
1177                 /* Ensure response is updated before thread context starts */
1178                 smp_wmb();
1179
1180                 /* There wont be separate events for link change initiated from
1181                  * software; Hence report the command responses as events
1182                  */
1183                 if (cgx_cmdresp_is_linkevent(event))
1184                         cgx_link_change_handler(event, lmac);
1185
1186                 /* Release thread waiting for completion  */
1187                 lmac->cmd_pend = false;
1188                 wake_up_interruptible(&lmac->wq_cmd_cmplt);
1189                 break;
1190         case CGX_EVT_ASYNC:
1191                 if (cgx_event_is_linkevent(event))
1192                         cgx_link_change_handler(event, lmac);
1193                 break;
1194         }
1195
1196         /* Any new event or command response will be posted by firmware
1197          * only after the current status is acked.
1198          * Ack the interrupt register as well.
1199          */
1200         cgx_write(lmac->cgx, lmac->lmac_id, CGX_EVENT_REG, 0);
1201         cgx_write(lmac->cgx, lmac->lmac_id, offset, clear_bit);
1202
1203         return IRQ_HANDLED;
1204 }
1205
1206 /* APIs for PHY management using CGX firmware interface */
1207
1208 /* callback registration for hardware events like link change */
1209 int cgx_lmac_evh_register(struct cgx_event_cb *cb, void *cgxd, int lmac_id)
1210 {
1211         struct cgx *cgx = cgxd;
1212         struct lmac *lmac;
1213
1214         lmac = lmac_pdata(lmac_id, cgx);
1215         if (!lmac)
1216                 return -ENODEV;
1217
1218         lmac->event_cb = *cb;
1219
1220         return 0;
1221 }
1222
1223 int cgx_lmac_evh_unregister(void *cgxd, int lmac_id)
1224 {
1225         struct lmac *lmac;
1226         unsigned long flags;
1227         struct cgx *cgx = cgxd;
1228
1229         lmac = lmac_pdata(lmac_id, cgx);
1230         if (!lmac)
1231                 return -ENODEV;
1232
1233         spin_lock_irqsave(&lmac->event_cb_lock, flags);
1234         lmac->event_cb.notify_link_chg = NULL;
1235         lmac->event_cb.data = NULL;
1236         spin_unlock_irqrestore(&lmac->event_cb_lock, flags);
1237
1238         return 0;
1239 }
1240
1241 int cgx_get_fwdata_base(u64 *base)
1242 {
1243         u64 req = 0, resp;
1244         struct cgx *cgx;
1245         int first_lmac;
1246         int err;
1247
1248         cgx = list_first_entry_or_null(&cgx_list, struct cgx, cgx_list);
1249         if (!cgx)
1250                 return -ENXIO;
1251
1252         first_lmac = find_first_bit(&cgx->lmac_bmap, MAX_LMAC_PER_CGX);
1253         req = FIELD_SET(CMDREG_ID, CGX_CMD_GET_FWD_BASE, req);
1254         err = cgx_fwi_cmd_generic(req, &resp, cgx, first_lmac);
1255         if (!err)
1256                 *base = FIELD_GET(RESP_FWD_BASE, resp);
1257
1258         return err;
1259 }
1260
1261 int cgx_set_link_mode(void *cgxd, struct cgx_set_link_mode_args args,
1262                       int cgx_id, int lmac_id)
1263 {
1264         struct cgx *cgx = cgxd;
1265         u64 req = 0, resp;
1266
1267         if (!cgx)
1268                 return -ENODEV;
1269
1270         if (args.mode)
1271                 otx2_map_ethtool_link_modes(args.mode, &args);
1272         if (!args.speed && args.duplex && !args.an)
1273                 return -EINVAL;
1274
1275         req = FIELD_SET(CMDREG_ID, CGX_CMD_MODE_CHANGE, req);
1276         req = FIELD_SET(CMDMODECHANGE_SPEED,
1277                         cgx_link_usertable_index_map(args.speed), req);
1278         req = FIELD_SET(CMDMODECHANGE_DUPLEX, args.duplex, req);
1279         req = FIELD_SET(CMDMODECHANGE_AN, args.an, req);
1280         req = FIELD_SET(CMDMODECHANGE_PORT, args.ports, req);
1281         req = FIELD_SET(CMDMODECHANGE_FLAGS, args.mode, req);
1282
1283         return cgx_fwi_cmd_generic(req, &resp, cgx, lmac_id);
1284 }
1285 int cgx_set_fec(u64 fec, int cgx_id, int lmac_id)
1286 {
1287         u64 req = 0, resp;
1288         struct cgx *cgx;
1289         int err = 0;
1290
1291         cgx = cgx_get_pdata(cgx_id);
1292         if (!cgx)
1293                 return -ENXIO;
1294
1295         req = FIELD_SET(CMDREG_ID, CGX_CMD_SET_FEC, req);
1296         req = FIELD_SET(CMDSETFEC, fec, req);
1297         err = cgx_fwi_cmd_generic(req, &resp, cgx, lmac_id);
1298         if (err)
1299                 return err;
1300
1301         cgx->lmac_idmap[lmac_id]->link_info.fec =
1302                         FIELD_GET(RESP_LINKSTAT_FEC, resp);
1303         return cgx->lmac_idmap[lmac_id]->link_info.fec;
1304 }
1305
1306 int cgx_get_phy_fec_stats(void *cgxd, int lmac_id)
1307 {
1308         struct cgx *cgx = cgxd;
1309         u64 req = 0, resp;
1310
1311         if (!cgx)
1312                 return -ENODEV;
1313
1314         req = FIELD_SET(CMDREG_ID, CGX_CMD_GET_PHY_FEC_STATS, req);
1315         return cgx_fwi_cmd_generic(req, &resp, cgx, lmac_id);
1316 }
1317
1318 static int cgx_fwi_link_change(struct cgx *cgx, int lmac_id, bool enable)
1319 {
1320         u64 req = 0;
1321         u64 resp;
1322
1323         if (enable)
1324                 req = FIELD_SET(CMDREG_ID, CGX_CMD_LINK_BRING_UP, req);
1325         else
1326                 req = FIELD_SET(CMDREG_ID, CGX_CMD_LINK_BRING_DOWN, req);
1327
1328         return cgx_fwi_cmd_generic(req, &resp, cgx, lmac_id);
1329 }
1330
1331 static inline int cgx_fwi_read_version(u64 *resp, struct cgx *cgx)
1332 {
1333         int first_lmac = find_first_bit(&cgx->lmac_bmap, MAX_LMAC_PER_CGX);
1334         u64 req = 0;
1335
1336         req = FIELD_SET(CMDREG_ID, CGX_CMD_GET_FW_VER, req);
1337         return cgx_fwi_cmd_generic(req, resp, cgx, first_lmac);
1338 }
1339
1340 static int cgx_lmac_verify_fwi_version(struct cgx *cgx)
1341 {
1342         struct device *dev = &cgx->pdev->dev;
1343         int major_ver, minor_ver;
1344         u64 resp;
1345         int err;
1346
1347         if (!cgx->lmac_count)
1348                 return 0;
1349
1350         err = cgx_fwi_read_version(&resp, cgx);
1351         if (err)
1352                 return err;
1353
1354         major_ver = FIELD_GET(RESP_MAJOR_VER, resp);
1355         minor_ver = FIELD_GET(RESP_MINOR_VER, resp);
1356         dev_dbg(dev, "Firmware command interface version = %d.%d\n",
1357                 major_ver, minor_ver);
1358         if (major_ver != CGX_FIRMWARE_MAJOR_VER)
1359                 return -EIO;
1360         else
1361                 return 0;
1362 }
1363
1364 static void cgx_lmac_linkup_work(struct work_struct *work)
1365 {
1366         struct cgx *cgx = container_of(work, struct cgx, cgx_cmd_work);
1367         struct device *dev = &cgx->pdev->dev;
1368         int i, err;
1369
1370         /* Do Link up for all the enabled lmacs */
1371         for_each_set_bit(i, &cgx->lmac_bmap, MAX_LMAC_PER_CGX) {
1372                 err = cgx_fwi_link_change(cgx, i, true);
1373                 if (err)
1374                         dev_info(dev, "cgx port %d:%d Link up command failed\n",
1375                                  cgx->cgx_id, i);
1376         }
1377 }
1378
1379 int cgx_lmac_linkup_start(void *cgxd)
1380 {
1381         struct cgx *cgx = cgxd;
1382
1383         if (!cgx)
1384                 return -ENODEV;
1385
1386         queue_work(cgx->cgx_cmd_workq, &cgx->cgx_cmd_work);
1387
1388         return 0;
1389 }
1390
1391 static void cgx_lmac_get_fifolen(struct cgx *cgx)
1392 {
1393         u64 cfg;
1394
1395         cfg = cgx_read(cgx, 0, CGX_CONST);
1396         cgx->mac_ops->fifo_len = FIELD_GET(CGX_CONST_RXFIFO_SIZE, cfg);
1397 }
1398
1399 static int cgx_configure_interrupt(struct cgx *cgx, struct lmac *lmac,
1400                                    int cnt, bool req_free)
1401 {
1402         struct mac_ops *mac_ops = cgx->mac_ops;
1403         u64 offset, ena_bit;
1404         unsigned int irq;
1405         int err;
1406
1407         irq      = pci_irq_vector(cgx->pdev, mac_ops->lmac_fwi +
1408                                   cnt * mac_ops->irq_offset);
1409         offset   = mac_ops->int_set_reg;
1410         ena_bit  = mac_ops->int_ena_bit;
1411
1412         if (req_free) {
1413                 free_irq(irq, lmac);
1414                 return 0;
1415         }
1416
1417         err = request_irq(irq, cgx_fwi_event_handler, 0, lmac->name, lmac);
1418         if (err)
1419                 return err;
1420
1421         /* Enable interrupt */
1422         cgx_write(cgx, lmac->lmac_id, offset, ena_bit);
1423         return 0;
1424 }
1425
1426 int cgx_get_nr_lmacs(void *cgxd)
1427 {
1428         struct cgx *cgx = cgxd;
1429
1430         return cgx_read(cgx, 0, CGXX_CMRX_RX_LMACS) & 0x7ULL;
1431 }
1432
1433 u8 cgx_get_lmacid(void *cgxd, u8 lmac_index)
1434 {
1435         struct cgx *cgx = cgxd;
1436
1437         return cgx->lmac_idmap[lmac_index]->lmac_id;
1438 }
1439
1440 unsigned long cgx_get_lmac_bmap(void *cgxd)
1441 {
1442         struct cgx *cgx = cgxd;
1443
1444         return cgx->lmac_bmap;
1445 }
1446
1447 static int cgx_lmac_init(struct cgx *cgx)
1448 {
1449         struct lmac *lmac;
1450         u64 lmac_list;
1451         int i, err;
1452
1453         cgx_lmac_get_fifolen(cgx);
1454
1455         cgx->lmac_count = cgx->mac_ops->get_nr_lmacs(cgx);
1456         /* lmac_list specifies which lmacs are enabled
1457          * when bit n is set to 1, LMAC[n] is enabled
1458          */
1459         if (cgx->mac_ops->non_contiguous_serdes_lane)
1460                 lmac_list = cgx_read(cgx, 0, CGXX_CMRX_RX_LMACS) & 0xFULL;
1461
1462         if (cgx->lmac_count > MAX_LMAC_PER_CGX)
1463                 cgx->lmac_count = MAX_LMAC_PER_CGX;
1464
1465         for (i = 0; i < cgx->lmac_count; i++) {
1466                 lmac = kzalloc(sizeof(struct lmac), GFP_KERNEL);
1467                 if (!lmac)
1468                         return -ENOMEM;
1469                 lmac->name = kcalloc(1, sizeof("cgx_fwi_xxx_yyy"), GFP_KERNEL);
1470                 if (!lmac->name) {
1471                         err = -ENOMEM;
1472                         goto err_lmac_free;
1473                 }
1474                 sprintf(lmac->name, "cgx_fwi_%d_%d", cgx->cgx_id, i);
1475                 if (cgx->mac_ops->non_contiguous_serdes_lane) {
1476                         lmac->lmac_id = __ffs64(lmac_list);
1477                         lmac_list   &= ~BIT_ULL(lmac->lmac_id);
1478                 } else {
1479                         lmac->lmac_id = i;
1480                 }
1481
1482                 lmac->cgx = cgx;
1483                 lmac->mac_to_index_bmap.max =
1484                                 MAX_DMAC_ENTRIES_PER_CGX / cgx->lmac_count;
1485                 err = rvu_alloc_bitmap(&lmac->mac_to_index_bmap);
1486                 if (err)
1487                         goto err_name_free;
1488
1489                 /* Reserve first entry for default MAC address */
1490                 set_bit(0, lmac->mac_to_index_bmap.bmap);
1491
1492                 init_waitqueue_head(&lmac->wq_cmd_cmplt);
1493                 mutex_init(&lmac->cmd_lock);
1494                 spin_lock_init(&lmac->event_cb_lock);
1495                 err = cgx_configure_interrupt(cgx, lmac, lmac->lmac_id, false);
1496                 if (err)
1497                         goto err_bitmap_free;
1498
1499                 /* Add reference */
1500                 cgx->lmac_idmap[lmac->lmac_id] = lmac;
1501                 set_bit(lmac->lmac_id, &cgx->lmac_bmap);
1502                 cgx->mac_ops->mac_pause_frm_config(cgx, lmac->lmac_id, true);
1503         }
1504
1505         return cgx_lmac_verify_fwi_version(cgx);
1506
1507 err_bitmap_free:
1508         rvu_free_bitmap(&lmac->mac_to_index_bmap);
1509 err_name_free:
1510         kfree(lmac->name);
1511 err_lmac_free:
1512         kfree(lmac);
1513         return err;
1514 }
1515
1516 static int cgx_lmac_exit(struct cgx *cgx)
1517 {
1518         struct lmac *lmac;
1519         int i;
1520
1521         if (cgx->cgx_cmd_workq) {
1522                 destroy_workqueue(cgx->cgx_cmd_workq);
1523                 cgx->cgx_cmd_workq = NULL;
1524         }
1525
1526         /* Free all lmac related resources */
1527         for_each_set_bit(i, &cgx->lmac_bmap, MAX_LMAC_PER_CGX) {
1528                 lmac = cgx->lmac_idmap[i];
1529                 if (!lmac)
1530                         continue;
1531                 cgx->mac_ops->mac_pause_frm_config(cgx, lmac->lmac_id, false);
1532                 cgx_configure_interrupt(cgx, lmac, lmac->lmac_id, true);
1533                 kfree(lmac->mac_to_index_bmap.bmap);
1534                 kfree(lmac->name);
1535                 kfree(lmac);
1536         }
1537
1538         return 0;
1539 }
1540
1541 static void cgx_populate_features(struct cgx *cgx)
1542 {
1543         if (is_dev_rpm(cgx))
1544                 cgx->hw_features = (RVU_LMAC_FEAT_DMACF | RVU_MAC_RPM |
1545                                     RVU_LMAC_FEAT_FC | RVU_LMAC_FEAT_PTP);
1546         else
1547                 cgx->hw_features = (RVU_LMAC_FEAT_FC  | RVU_LMAC_FEAT_HIGIG2 |
1548                                     RVU_LMAC_FEAT_PTP | RVU_LMAC_FEAT_DMACF);
1549 }
1550
1551 static struct mac_ops   cgx_mac_ops    = {
1552         .name           =       "cgx",
1553         .csr_offset     =       0,
1554         .lmac_offset    =       18,
1555         .int_register   =       CGXX_CMRX_INT,
1556         .int_set_reg    =       CGXX_CMRX_INT_ENA_W1S,
1557         .irq_offset     =       9,
1558         .int_ena_bit    =       FW_CGX_INT,
1559         .lmac_fwi       =       CGX_LMAC_FWI,
1560         .non_contiguous_serdes_lane = false,
1561         .rx_stats_cnt   =       9,
1562         .tx_stats_cnt   =       18,
1563         .get_nr_lmacs   =       cgx_get_nr_lmacs,
1564         .get_lmac_type  =       cgx_get_lmac_type,
1565         .mac_lmac_intl_lbk =    cgx_lmac_internal_loopback,
1566         .mac_get_rx_stats  =    cgx_get_rx_stats,
1567         .mac_get_tx_stats  =    cgx_get_tx_stats,
1568         .mac_enadis_rx_pause_fwding =   cgx_lmac_enadis_rx_pause_fwding,
1569         .mac_get_pause_frm_status =     cgx_lmac_get_pause_frm_status,
1570         .mac_enadis_pause_frm =         cgx_lmac_enadis_pause_frm,
1571         .mac_pause_frm_config =         cgx_lmac_pause_frm_config,
1572         .mac_enadis_ptp_config =        cgx_lmac_ptp_config,
1573         .mac_rx_tx_enable =             cgx_lmac_rx_tx_enable,
1574         .mac_tx_enable =                cgx_lmac_tx_enable,
1575 };
1576
1577 static int cgx_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1578 {
1579         struct device *dev = &pdev->dev;
1580         struct cgx *cgx;
1581         int err, nvec;
1582
1583         cgx = devm_kzalloc(dev, sizeof(*cgx), GFP_KERNEL);
1584         if (!cgx)
1585                 return -ENOMEM;
1586         cgx->pdev = pdev;
1587
1588         pci_set_drvdata(pdev, cgx);
1589
1590         /* Use mac_ops to get MAC specific features */
1591         if (pdev->device == PCI_DEVID_CN10K_RPM)
1592                 cgx->mac_ops = rpm_get_mac_ops();
1593         else
1594                 cgx->mac_ops = &cgx_mac_ops;
1595
1596         err = pci_enable_device(pdev);
1597         if (err) {
1598                 dev_err(dev, "Failed to enable PCI device\n");
1599                 pci_set_drvdata(pdev, NULL);
1600                 return err;
1601         }
1602
1603         err = pci_request_regions(pdev, DRV_NAME);
1604         if (err) {
1605                 dev_err(dev, "PCI request regions failed 0x%x\n", err);
1606                 goto err_disable_device;
1607         }
1608
1609         /* MAP configuration registers */
1610         cgx->reg_base = pcim_iomap(pdev, PCI_CFG_REG_BAR_NUM, 0);
1611         if (!cgx->reg_base) {
1612                 dev_err(dev, "CGX: Cannot map CSR memory space, aborting\n");
1613                 err = -ENOMEM;
1614                 goto err_release_regions;
1615         }
1616
1617         nvec = pci_msix_vec_count(cgx->pdev);
1618         err = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX);
1619         if (err < 0 || err != nvec) {
1620                 dev_err(dev, "Request for %d msix vectors failed, err %d\n",
1621                         nvec, err);
1622                 goto err_release_regions;
1623         }
1624
1625         cgx->cgx_id = (pci_resource_start(pdev, PCI_CFG_REG_BAR_NUM) >> 24)
1626                 & CGX_ID_MASK;
1627
1628         /* init wq for processing linkup requests */
1629         INIT_WORK(&cgx->cgx_cmd_work, cgx_lmac_linkup_work);
1630         cgx->cgx_cmd_workq = alloc_workqueue("cgx_cmd_workq", 0, 0);
1631         if (!cgx->cgx_cmd_workq) {
1632                 dev_err(dev, "alloc workqueue failed for cgx cmd");
1633                 err = -ENOMEM;
1634                 goto err_free_irq_vectors;
1635         }
1636
1637         list_add(&cgx->cgx_list, &cgx_list);
1638
1639
1640         cgx_populate_features(cgx);
1641
1642         mutex_init(&cgx->lock);
1643
1644         err = cgx_lmac_init(cgx);
1645         if (err)
1646                 goto err_release_lmac;
1647
1648         return 0;
1649
1650 err_release_lmac:
1651         cgx_lmac_exit(cgx);
1652         list_del(&cgx->cgx_list);
1653 err_free_irq_vectors:
1654         pci_free_irq_vectors(pdev);
1655 err_release_regions:
1656         pci_release_regions(pdev);
1657 err_disable_device:
1658         pci_disable_device(pdev);
1659         pci_set_drvdata(pdev, NULL);
1660         return err;
1661 }
1662
1663 static void cgx_remove(struct pci_dev *pdev)
1664 {
1665         struct cgx *cgx = pci_get_drvdata(pdev);
1666
1667         if (cgx) {
1668                 cgx_lmac_exit(cgx);
1669                 list_del(&cgx->cgx_list);
1670         }
1671         pci_free_irq_vectors(pdev);
1672         pci_release_regions(pdev);
1673         pci_disable_device(pdev);
1674         pci_set_drvdata(pdev, NULL);
1675 }
1676
1677 struct pci_driver cgx_driver = {
1678         .name = DRV_NAME,
1679         .id_table = cgx_id_table,
1680         .probe = cgx_probe,
1681         .remove = cgx_remove,
1682 };